Annotation of src/sys/uvm/uvm_aobj.c, Revision 1.48
1.48 ! chs 1: /* $NetBSD: uvm_aobj.c,v 1.47 2001/11/06 08:07:49 chs Exp $ */
1.6 mrg 2:
1.7 chs 3: /*
4: * Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
5: * Washington University.
6: * All rights reserved.
7: *
8: * Redistribution and use in source and binary forms, with or without
9: * modification, are permitted provided that the following conditions
10: * are met:
11: * 1. Redistributions of source code must retain the above copyright
12: * notice, this list of conditions and the following disclaimer.
13: * 2. Redistributions in binary form must reproduce the above copyright
14: * notice, this list of conditions and the following disclaimer in the
15: * documentation and/or other materials provided with the distribution.
16: * 3. All advertising materials mentioning features or use of this software
17: * must display the following acknowledgement:
18: * This product includes software developed by Charles D. Cranor and
19: * Washington University.
20: * 4. The name of the author may not be used to endorse or promote products
21: * derived from this software without specific prior written permission.
22: *
23: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33: *
1.4 mrg 34: * from: Id: uvm_aobj.c,v 1.1.2.5 1998/02/06 05:14:38 chs Exp
35: */
1.7 chs 36: /*
37: * uvm_aobj.c: anonymous memory uvm_object pager
38: *
39: * author: Chuck Silvers <chuq@chuq.com>
40: * started: Jan-1998
41: *
42: * - design mostly from Chuck Cranor
43: */
44:
45: #include "opt_uvmhist.h"
1.1 mrg 46:
47: #include <sys/param.h>
48: #include <sys/systm.h>
49: #include <sys/proc.h>
50: #include <sys/malloc.h>
1.37 chs 51: #include <sys/kernel.h>
1.12 thorpej 52: #include <sys/pool.h>
1.27 chs 53: #include <sys/kernel.h>
1.1 mrg 54:
55: #include <uvm/uvm.h>
56:
57: /*
58: * an aobj manages anonymous-memory backed uvm_objects. in addition
59: * to keeping the list of resident pages, it also keeps a list of
60: * allocated swap blocks. depending on the size of the aobj this list
61: * of allocated swap blocks is either stored in an array (small objects)
62: * or in a hash table (large objects).
63: */
64:
65: /*
66: * local structures
67: */
68:
69: /*
70: * for hash tables, we break the address space of the aobj into blocks
71: * of UAO_SWHASH_CLUSTER_SIZE pages. we require the cluster size to
72: * be a power of two.
73: */
74:
75: #define UAO_SWHASH_CLUSTER_SHIFT 4
76: #define UAO_SWHASH_CLUSTER_SIZE (1 << UAO_SWHASH_CLUSTER_SHIFT)
77:
78: /* get the "tag" for this page index */
79: #define UAO_SWHASH_ELT_TAG(PAGEIDX) \
80: ((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT)
81:
82: /* given an ELT and a page index, find the swap slot */
83: #define UAO_SWHASH_ELT_PAGESLOT(ELT, PAGEIDX) \
84: ((ELT)->slots[(PAGEIDX) & (UAO_SWHASH_CLUSTER_SIZE - 1)])
85:
86: /* given an ELT, return its pageidx base */
87: #define UAO_SWHASH_ELT_PAGEIDX_BASE(ELT) \
88: ((ELT)->tag << UAO_SWHASH_CLUSTER_SHIFT)
89:
90: /*
91: * the swhash hash function
92: */
1.46 chs 93:
1.1 mrg 94: #define UAO_SWHASH_HASH(AOBJ, PAGEIDX) \
95: (&(AOBJ)->u_swhash[(((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT) \
96: & (AOBJ)->u_swhashmask)])
97:
98: /*
99: * the swhash threshhold determines if we will use an array or a
100: * hash table to store the list of allocated swap blocks.
101: */
102:
103: #define UAO_SWHASH_THRESHOLD (UAO_SWHASH_CLUSTER_SIZE * 4)
104: #define UAO_USES_SWHASH(AOBJ) \
105: ((AOBJ)->u_pages > UAO_SWHASH_THRESHOLD) /* use hash? */
106:
107: /*
1.3 chs 108: * the number of buckets in a swhash, with an upper bound
1.1 mrg 109: */
1.46 chs 110:
1.1 mrg 111: #define UAO_SWHASH_MAXBUCKETS 256
112: #define UAO_SWHASH_BUCKETS(AOBJ) \
1.46 chs 113: (MIN((AOBJ)->u_pages >> UAO_SWHASH_CLUSTER_SHIFT, \
1.1 mrg 114: UAO_SWHASH_MAXBUCKETS))
115:
116:
117: /*
118: * uao_swhash_elt: when a hash table is being used, this structure defines
119: * the format of an entry in the bucket list.
120: */
121:
122: struct uao_swhash_elt {
1.5 mrg 123: LIST_ENTRY(uao_swhash_elt) list; /* the hash list */
1.28 kleink 124: voff_t tag; /* our 'tag' */
1.5 mrg 125: int count; /* our number of active slots */
126: int slots[UAO_SWHASH_CLUSTER_SIZE]; /* the slots */
1.1 mrg 127: };
128:
129: /*
130: * uao_swhash: the swap hash table structure
131: */
132:
133: LIST_HEAD(uao_swhash, uao_swhash_elt);
134:
1.12 thorpej 135: /*
136: * uao_swhash_elt_pool: pool of uao_swhash_elt structures
137: */
138:
139: struct pool uao_swhash_elt_pool;
1.1 mrg 140:
141: /*
142: * uvm_aobj: the actual anon-backed uvm_object
143: *
144: * => the uvm_object is at the top of the structure, this allows
1.46 chs 145: * (struct uvm_aobj *) == (struct uvm_object *)
1.1 mrg 146: * => only one of u_swslots and u_swhash is used in any given aobj
147: */
148:
149: struct uvm_aobj {
1.5 mrg 150: struct uvm_object u_obj; /* has: lock, pgops, memq, #pages, #refs */
1.11 drochner 151: int u_pages; /* number of pages in entire object */
1.5 mrg 152: int u_flags; /* the flags (see uvm_aobj.h) */
153: int *u_swslots; /* array of offset->swapslot mappings */
154: /*
155: * hashtable of offset->swapslot mappings
156: * (u_swhash is an array of bucket heads)
157: */
158: struct uao_swhash *u_swhash;
159: u_long u_swhashmask; /* mask for hashtable */
160: LIST_ENTRY(uvm_aobj) u_list; /* global list of aobjs */
1.1 mrg 161: };
162:
163: /*
1.12 thorpej 164: * uvm_aobj_pool: pool of uvm_aobj structures
165: */
166:
167: struct pool uvm_aobj_pool;
168:
169: /*
1.1 mrg 170: * local functions
171: */
172:
1.46 chs 173: static struct uao_swhash_elt *uao_find_swhash_elt
174: __P((struct uvm_aobj *, int, boolean_t));
175:
176: static void uao_free __P((struct uvm_aobj *));
177: static int uao_get __P((struct uvm_object *, voff_t, struct vm_page **,
178: int *, int, vm_prot_t, int, int));
179: static boolean_t uao_put __P((struct uvm_object *, voff_t, voff_t, int));
180: static boolean_t uao_pagein __P((struct uvm_aobj *, int, int));
181: static boolean_t uao_pagein_page __P((struct uvm_aobj *, int));
1.1 mrg 182:
183: /*
184: * aobj_pager
1.41 chs 185: *
1.1 mrg 186: * note that some functions (e.g. put) are handled elsewhere
187: */
188:
189: struct uvm_pagerops aobj_pager = {
1.27 chs 190: NULL, /* init */
1.5 mrg 191: uao_reference, /* reference */
192: uao_detach, /* detach */
193: NULL, /* fault */
194: uao_get, /* get */
1.46 chs 195: uao_put, /* flush */
1.1 mrg 196: };
197:
198: /*
199: * uao_list: global list of active aobjs, locked by uao_list_lock
200: */
201:
202: static LIST_HEAD(aobjlist, uvm_aobj) uao_list;
1.42 chs 203: static struct simplelock uao_list_lock;
1.1 mrg 204:
205: /*
206: * functions
207: */
208:
209: /*
210: * hash table/array related functions
211: */
212:
213: /*
214: * uao_find_swhash_elt: find (or create) a hash table entry for a page
215: * offset.
216: *
217: * => the object should be locked by the caller
218: */
219:
1.5 mrg 220: static struct uao_swhash_elt *
221: uao_find_swhash_elt(aobj, pageidx, create)
222: struct uvm_aobj *aobj;
223: int pageidx;
224: boolean_t create;
225: {
226: struct uao_swhash *swhash;
227: struct uao_swhash_elt *elt;
1.28 kleink 228: voff_t page_tag;
1.1 mrg 229:
1.45 chs 230: swhash = UAO_SWHASH_HASH(aobj, pageidx);
231: page_tag = UAO_SWHASH_ELT_TAG(pageidx);
1.1 mrg 232:
1.5 mrg 233: /*
234: * now search the bucket for the requested tag
235: */
1.45 chs 236:
1.37 chs 237: LIST_FOREACH(elt, swhash, list) {
1.45 chs 238: if (elt->tag == page_tag) {
239: return elt;
240: }
1.5 mrg 241: }
1.45 chs 242: if (!create) {
1.5 mrg 243: return NULL;
1.45 chs 244: }
1.5 mrg 245:
246: /*
1.12 thorpej 247: * allocate a new entry for the bucket and init/insert it in
1.5 mrg 248: */
1.45 chs 249:
250: elt = pool_get(&uao_swhash_elt_pool, PR_NOWAIT);
251: if (elt == NULL) {
252: return NULL;
253: }
1.5 mrg 254: LIST_INSERT_HEAD(swhash, elt, list);
255: elt->tag = page_tag;
256: elt->count = 0;
1.9 perry 257: memset(elt->slots, 0, sizeof(elt->slots));
1.45 chs 258: return elt;
1.1 mrg 259: }
260:
261: /*
262: * uao_find_swslot: find the swap slot number for an aobj/pageidx
263: *
1.41 chs 264: * => object must be locked by caller
1.1 mrg 265: */
1.46 chs 266:
267: int
268: uao_find_swslot(uobj, pageidx)
269: struct uvm_object *uobj;
1.11 drochner 270: int pageidx;
1.1 mrg 271: {
1.46 chs 272: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
273: struct uao_swhash_elt *elt;
1.1 mrg 274:
1.5 mrg 275: /*
276: * if noswap flag is set, then we never return a slot
277: */
1.1 mrg 278:
1.5 mrg 279: if (aobj->u_flags & UAO_FLAG_NOSWAP)
280: return(0);
1.1 mrg 281:
1.5 mrg 282: /*
283: * if hashing, look in hash table.
284: */
1.1 mrg 285:
1.5 mrg 286: if (UAO_USES_SWHASH(aobj)) {
1.46 chs 287: elt = uao_find_swhash_elt(aobj, pageidx, FALSE);
1.5 mrg 288: if (elt)
289: return(UAO_SWHASH_ELT_PAGESLOT(elt, pageidx));
290: else
1.31 thorpej 291: return(0);
1.5 mrg 292: }
1.1 mrg 293:
1.41 chs 294: /*
1.5 mrg 295: * otherwise, look in the array
296: */
1.46 chs 297:
1.5 mrg 298: return(aobj->u_swslots[pageidx]);
1.1 mrg 299: }
300:
301: /*
302: * uao_set_swslot: set the swap slot for a page in an aobj.
303: *
304: * => setting a slot to zero frees the slot
305: * => object must be locked by caller
1.45 chs 306: * => we return the old slot number, or -1 if we failed to allocate
307: * memory to record the new slot number
1.1 mrg 308: */
1.46 chs 309:
1.5 mrg 310: int
311: uao_set_swslot(uobj, pageidx, slot)
312: struct uvm_object *uobj;
313: int pageidx, slot;
314: {
315: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.45 chs 316: struct uao_swhash_elt *elt;
1.5 mrg 317: int oldslot;
318: UVMHIST_FUNC("uao_set_swslot"); UVMHIST_CALLED(pdhist);
319: UVMHIST_LOG(pdhist, "aobj %p pageidx %d slot %d",
320: aobj, pageidx, slot, 0);
1.1 mrg 321:
1.5 mrg 322: /*
1.46 chs 323: * if noswap flag is set, then we can't set a non-zero slot.
1.5 mrg 324: */
1.1 mrg 325:
1.5 mrg 326: if (aobj->u_flags & UAO_FLAG_NOSWAP) {
327: if (slot == 0)
1.46 chs 328: return(0);
1.1 mrg 329:
1.5 mrg 330: printf("uao_set_swslot: uobj = %p\n", uobj);
1.46 chs 331: panic("uao_set_swslot: NOSWAP object");
1.5 mrg 332: }
1.1 mrg 333:
1.5 mrg 334: /*
335: * are we using a hash table? if so, add it in the hash.
336: */
1.1 mrg 337:
1.5 mrg 338: if (UAO_USES_SWHASH(aobj)) {
1.39 chs 339:
1.12 thorpej 340: /*
341: * Avoid allocating an entry just to free it again if
342: * the page had not swap slot in the first place, and
343: * we are freeing.
344: */
1.39 chs 345:
1.46 chs 346: elt = uao_find_swhash_elt(aobj, pageidx, slot != 0);
1.12 thorpej 347: if (elt == NULL) {
1.45 chs 348: return slot ? -1 : 0;
1.12 thorpej 349: }
1.5 mrg 350:
351: oldslot = UAO_SWHASH_ELT_PAGESLOT(elt, pageidx);
352: UAO_SWHASH_ELT_PAGESLOT(elt, pageidx) = slot;
353:
354: /*
355: * now adjust the elt's reference counter and free it if we've
356: * dropped it to zero.
357: */
358:
359: if (slot) {
360: if (oldslot == 0)
361: elt->count++;
1.45 chs 362: } else {
363: if (oldslot)
1.5 mrg 364: elt->count--;
365:
366: if (elt->count == 0) {
367: LIST_REMOVE(elt, list);
1.12 thorpej 368: pool_put(&uao_swhash_elt_pool, elt);
1.5 mrg 369: }
370: }
1.41 chs 371: } else {
1.5 mrg 372: /* we are using an array */
373: oldslot = aobj->u_swslots[pageidx];
374: aobj->u_swslots[pageidx] = slot;
375: }
376: return (oldslot);
1.1 mrg 377: }
378:
379: /*
380: * end of hash/array functions
381: */
382:
383: /*
384: * uao_free: free all resources held by an aobj, and then free the aobj
385: *
386: * => the aobj should be dead
387: */
1.46 chs 388:
1.1 mrg 389: static void
390: uao_free(aobj)
1.5 mrg 391: struct uvm_aobj *aobj;
1.1 mrg 392: {
1.46 chs 393: int swpgonlydelta = 0;
1.1 mrg 394:
1.27 chs 395: simple_unlock(&aobj->u_obj.vmobjlock);
1.5 mrg 396: if (UAO_USES_SWHASH(aobj)) {
397: int i, hashbuckets = aobj->u_swhashmask + 1;
1.1 mrg 398:
1.5 mrg 399: /*
400: * free the swslots from each hash bucket,
401: * then the hash bucket, and finally the hash table itself.
402: */
1.46 chs 403:
1.5 mrg 404: for (i = 0; i < hashbuckets; i++) {
405: struct uao_swhash_elt *elt, *next;
406:
1.27 chs 407: for (elt = LIST_FIRST(&aobj->u_swhash[i]);
408: elt != NULL;
409: elt = next) {
1.5 mrg 410: int j;
411:
1.27 chs 412: for (j = 0; j < UAO_SWHASH_CLUSTER_SIZE; j++) {
1.5 mrg 413: int slot = elt->slots[j];
414:
1.37 chs 415: if (slot == 0) {
416: continue;
417: }
418: uvm_swap_free(slot, 1);
1.46 chs 419: swpgonlydelta++;
1.5 mrg 420: }
421:
1.27 chs 422: next = LIST_NEXT(elt, list);
1.12 thorpej 423: pool_put(&uao_swhash_elt_pool, elt);
1.5 mrg 424: }
425: }
1.34 thorpej 426: free(aobj->u_swhash, M_UVMAOBJ);
1.5 mrg 427: } else {
428: int i;
429:
430: /*
431: * free the array
432: */
433:
1.27 chs 434: for (i = 0; i < aobj->u_pages; i++) {
1.5 mrg 435: int slot = aobj->u_swslots[i];
436:
1.18 chs 437: if (slot) {
1.5 mrg 438: uvm_swap_free(slot, 1);
1.46 chs 439: swpgonlydelta++;
1.18 chs 440: }
1.5 mrg 441: }
1.34 thorpej 442: free(aobj->u_swslots, M_UVMAOBJ);
1.1 mrg 443: }
444:
1.5 mrg 445: /*
446: * finally free the aobj itself
447: */
1.46 chs 448:
1.12 thorpej 449: pool_put(&uvm_aobj_pool, aobj);
1.46 chs 450:
451: /*
452: * adjust the counter of pages only in swap for all
453: * the swap slots we've freed.
454: */
455:
1.48 ! chs 456: if (swpgonlydelta > 0) {
! 457: simple_lock(&uvm.swap_data_lock);
! 458: KASSERT(uvmexp.swpgonly >= swpgonlydelta);
! 459: uvmexp.swpgonly -= swpgonlydelta;
! 460: simple_unlock(&uvm.swap_data_lock);
! 461: }
1.1 mrg 462: }
463:
464: /*
465: * pager functions
466: */
467:
468: /*
469: * uao_create: create an aobj of the given size and return its uvm_object.
470: *
471: * => for normal use, flags are always zero
472: * => for the kernel object, the flags are:
473: * UAO_FLAG_KERNOBJ - allocate the kernel object (can only happen once)
474: * UAO_FLAG_KERNSWAP - enable swapping of kernel object (" ")
475: */
1.46 chs 476:
1.5 mrg 477: struct uvm_object *
478: uao_create(size, flags)
1.10 eeh 479: vsize_t size;
1.5 mrg 480: int flags;
481: {
1.46 chs 482: static struct uvm_aobj kernel_object_store;
483: static int kobj_alloced = 0;
1.15 chs 484: int pages = round_page(size) >> PAGE_SHIFT;
1.5 mrg 485: struct uvm_aobj *aobj;
1.1 mrg 486:
1.5 mrg 487: /*
1.27 chs 488: * malloc a new aobj unless we are asked for the kernel object
489: */
1.5 mrg 490:
1.46 chs 491: if (flags & UAO_FLAG_KERNOBJ) {
492: KASSERT(!kobj_alloced);
1.5 mrg 493: aobj = &kernel_object_store;
494: aobj->u_pages = pages;
1.46 chs 495: aobj->u_flags = UAO_FLAG_NOSWAP;
1.5 mrg 496: aobj->u_obj.uo_refs = UVM_OBJ_KERN;
497: kobj_alloced = UAO_FLAG_KERNOBJ;
498: } else if (flags & UAO_FLAG_KERNSWAP) {
1.46 chs 499: KASSERT(kobj_alloced == UAO_FLAG_KERNOBJ);
1.5 mrg 500: aobj = &kernel_object_store;
501: kobj_alloced = UAO_FLAG_KERNSWAP;
1.46 chs 502: } else {
1.12 thorpej 503: aobj = pool_get(&uvm_aobj_pool, PR_WAITOK);
1.5 mrg 504: aobj->u_pages = pages;
1.46 chs 505: aobj->u_flags = 0;
506: aobj->u_obj.uo_refs = 1;
1.5 mrg 507: }
1.1 mrg 508:
1.5 mrg 509: /*
510: * allocate hash/array if necessary
511: *
512: * note: in the KERNSWAP case no need to worry about locking since
513: * we are still booting we should be the only thread around.
514: */
1.46 chs 515:
1.5 mrg 516: if (flags == 0 || (flags & UAO_FLAG_KERNSWAP) != 0) {
517: int mflags = (flags & UAO_FLAG_KERNSWAP) != 0 ?
518: M_NOWAIT : M_WAITOK;
519:
520: /* allocate hash table or array depending on object size */
1.27 chs 521: if (UAO_USES_SWHASH(aobj)) {
1.5 mrg 522: aobj->u_swhash = hashinit(UAO_SWHASH_BUCKETS(aobj),
1.35 ad 523: HASH_LIST, M_UVMAOBJ, mflags, &aobj->u_swhashmask);
1.5 mrg 524: if (aobj->u_swhash == NULL)
525: panic("uao_create: hashinit swhash failed");
526: } else {
1.34 thorpej 527: aobj->u_swslots = malloc(pages * sizeof(int),
1.5 mrg 528: M_UVMAOBJ, mflags);
529: if (aobj->u_swslots == NULL)
530: panic("uao_create: malloc swslots failed");
1.9 perry 531: memset(aobj->u_swslots, 0, pages * sizeof(int));
1.5 mrg 532: }
533:
534: if (flags) {
535: aobj->u_flags &= ~UAO_FLAG_NOSWAP; /* clear noswap */
536: return(&aobj->u_obj);
537: }
538: }
539:
540: /*
541: * init aobj fields
542: */
1.46 chs 543:
1.5 mrg 544: simple_lock_init(&aobj->u_obj.vmobjlock);
545: aobj->u_obj.pgops = &aobj_pager;
546: TAILQ_INIT(&aobj->u_obj.memq);
547: aobj->u_obj.uo_npages = 0;
1.1 mrg 548:
1.5 mrg 549: /*
550: * now that aobj is ready, add it to the global list
551: */
1.46 chs 552:
1.5 mrg 553: simple_lock(&uao_list_lock);
554: LIST_INSERT_HEAD(&uao_list, aobj, u_list);
555: simple_unlock(&uao_list_lock);
556: return(&aobj->u_obj);
1.1 mrg 557: }
558:
559:
560:
561: /*
562: * uao_init: set up aobj pager subsystem
563: *
564: * => called at boot time from uvm_pager_init()
565: */
1.46 chs 566:
1.27 chs 567: void
1.46 chs 568: uao_init(void)
1.5 mrg 569: {
1.12 thorpej 570: static int uao_initialized;
571:
572: if (uao_initialized)
573: return;
574: uao_initialized = TRUE;
1.5 mrg 575: LIST_INIT(&uao_list);
576: simple_lock_init(&uao_list_lock);
1.12 thorpej 577:
1.14 thorpej 578: /*
579: * NOTE: Pages fror this pool must not come from a pageable
580: * kernel map!
581: */
1.46 chs 582:
1.12 thorpej 583: pool_init(&uao_swhash_elt_pool, sizeof(struct uao_swhash_elt),
1.13 thorpej 584: 0, 0, 0, "uaoeltpl", 0, NULL, NULL, M_UVMAOBJ);
1.12 thorpej 585: pool_init(&uvm_aobj_pool, sizeof(struct uvm_aobj), 0, 0, 0,
586: "aobjpl", 0,
587: pool_page_alloc_nointr, pool_page_free_nointr, M_UVMAOBJ);
1.1 mrg 588: }
589:
590: /*
591: * uao_reference: add a ref to an aobj
592: *
1.27 chs 593: * => aobj must be unlocked
594: * => just lock it and call the locked version
1.1 mrg 595: */
1.46 chs 596:
1.5 mrg 597: void
598: uao_reference(uobj)
599: struct uvm_object *uobj;
1.1 mrg 600: {
1.27 chs 601: simple_lock(&uobj->vmobjlock);
602: uao_reference_locked(uobj);
603: simple_unlock(&uobj->vmobjlock);
604: }
605:
606: /*
607: * uao_reference_locked: add a ref to an aobj that is already locked
608: *
609: * => aobj must be locked
610: * this needs to be separate from the normal routine
611: * since sometimes we need to add a reference to an aobj when
612: * it's already locked.
613: */
1.46 chs 614:
1.27 chs 615: void
616: uao_reference_locked(uobj)
617: struct uvm_object *uobj;
618: {
1.5 mrg 619: UVMHIST_FUNC("uao_reference"); UVMHIST_CALLED(maphist);
1.1 mrg 620:
1.5 mrg 621: /*
622: * kernel_object already has plenty of references, leave it alone.
623: */
1.1 mrg 624:
1.20 thorpej 625: if (UVM_OBJ_IS_KERN_OBJECT(uobj))
1.5 mrg 626: return;
1.1 mrg 627:
1.46 chs 628: uobj->uo_refs++;
1.41 chs 629: UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)",
1.27 chs 630: uobj, uobj->uo_refs,0,0);
1.1 mrg 631: }
632:
633: /*
634: * uao_detach: drop a reference to an aobj
635: *
1.27 chs 636: * => aobj must be unlocked
637: * => just lock it and call the locked version
1.1 mrg 638: */
1.46 chs 639:
1.5 mrg 640: void
641: uao_detach(uobj)
642: struct uvm_object *uobj;
643: {
1.27 chs 644: simple_lock(&uobj->vmobjlock);
645: uao_detach_locked(uobj);
646: }
647:
648: /*
649: * uao_detach_locked: drop a reference to an aobj
650: *
651: * => aobj must be locked, and is unlocked (or freed) upon return.
652: * this needs to be separate from the normal routine
653: * since sometimes we need to detach from an aobj when
654: * it's already locked.
655: */
1.46 chs 656:
1.27 chs 657: void
658: uao_detach_locked(uobj)
659: struct uvm_object *uobj;
660: {
1.5 mrg 661: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.46 chs 662: struct vm_page *pg;
1.5 mrg 663: UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist);
1.1 mrg 664:
1.5 mrg 665: /*
666: * detaching from kernel_object is a noop.
667: */
1.46 chs 668:
1.27 chs 669: if (UVM_OBJ_IS_KERN_OBJECT(uobj)) {
670: simple_unlock(&uobj->vmobjlock);
1.5 mrg 671: return;
1.27 chs 672: }
1.5 mrg 673:
674: UVMHIST_LOG(maphist," (uobj=0x%x) ref=%d", uobj,uobj->uo_refs,0,0);
1.46 chs 675: uobj->uo_refs--;
676: if (uobj->uo_refs) {
1.5 mrg 677: simple_unlock(&uobj->vmobjlock);
678: UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
679: return;
680: }
681:
682: /*
683: * remove the aobj from the global list.
684: */
1.46 chs 685:
1.5 mrg 686: simple_lock(&uao_list_lock);
687: LIST_REMOVE(aobj, u_list);
688: simple_unlock(&uao_list_lock);
689:
690: /*
1.46 chs 691: * free all the pages left in the aobj. for each page,
692: * when the page is no longer busy (and thus after any disk i/o that
693: * it's involved in is complete), release any swap resources and
694: * free the page itself.
1.5 mrg 695: */
1.46 chs 696:
697: uvm_lock_pageq();
698: while ((pg = TAILQ_FIRST(&uobj->memq)) != NULL) {
699: pmap_page_protect(pg, VM_PROT_NONE);
1.5 mrg 700: if (pg->flags & PG_BUSY) {
1.46 chs 701: pg->flags |= PG_WANTED;
702: uvm_unlock_pageq();
703: UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, FALSE,
704: "uao_det", 0);
705: simple_lock(&uobj->vmobjlock);
706: uvm_lock_pageq();
1.5 mrg 707: continue;
708: }
1.18 chs 709: uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
1.5 mrg 710: uvm_pagefree(pg);
711: }
1.46 chs 712: uvm_unlock_pageq();
1.1 mrg 713:
1.5 mrg 714: /*
1.46 chs 715: * finally, free the aobj itself.
1.5 mrg 716: */
1.1 mrg 717:
1.5 mrg 718: uao_free(aobj);
719: }
1.1 mrg 720:
721: /*
1.46 chs 722: * uao_put: flush pages out of a uvm object
1.22 thorpej 723: *
724: * => object should be locked by caller. we may _unlock_ the object
725: * if (and only if) we need to clean a page (PGO_CLEANIT).
726: * XXXJRT Currently, however, we don't. In the case of cleaning
727: * XXXJRT a page, we simply just deactivate it. Should probably
728: * XXXJRT handle this better, in the future (although "flushing"
729: * XXXJRT anonymous memory isn't terribly important).
730: * => if PGO_CLEANIT is not set, then we will neither unlock the object
731: * or block.
732: * => if PGO_ALLPAGE is set, then all pages in the object are valid targets
733: * for flushing.
734: * => NOTE: we rely on the fact that the object's memq is a TAILQ and
735: * that new pages are inserted on the tail end of the list. thus,
736: * we can make a complete pass through the object in one go by starting
737: * at the head and working towards the tail (new pages are put in
738: * front of us).
739: * => NOTE: we are allowed to lock the page queues, so the caller
740: * must not be holding the lock on them [e.g. pagedaemon had
741: * better not call us with the queues locked]
742: * => we return TRUE unless we encountered some sort of I/O error
743: * XXXJRT currently never happens, as we never directly initiate
744: * XXXJRT I/O
745: *
746: * note on page traversal:
747: * we can traverse the pages in an object either by going down the
748: * linked list in "uobj->memq", or we can go over the address range
749: * by page doing hash table lookups for each address. depending
750: * on how many pages are in the object it may be cheaper to do one
751: * or the other. we set "by_list" to true if we are using memq.
752: * if the cost of a hash lookup was equal to the cost of the list
753: * traversal we could compare the number of pages in the start->stop
754: * range to the total number of pages in the object. however, it
755: * seems that a hash table lookup is more expensive than the linked
756: * list traversal, so we multiply the number of pages in the
757: * start->stop range by a penalty which we define below.
1.1 mrg 758: */
1.22 thorpej 759:
1.46 chs 760: int
761: uao_put(uobj, start, stop, flags)
1.5 mrg 762: struct uvm_object *uobj;
1.28 kleink 763: voff_t start, stop;
1.5 mrg 764: int flags;
765: {
1.46 chs 766: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
767: struct vm_page *pg, *nextpg;
768: boolean_t by_list;
1.28 kleink 769: voff_t curoff;
1.46 chs 770: UVMHIST_FUNC("uao_put"); UVMHIST_CALLED(maphist);
1.22 thorpej 771:
1.46 chs 772: curoff = 0;
1.22 thorpej 773: if (flags & PGO_ALLPAGES) {
774: start = 0;
775: stop = aobj->u_pages << PAGE_SHIFT;
776: by_list = TRUE; /* always go by the list */
777: } else {
778: start = trunc_page(start);
779: stop = round_page(stop);
780: if (stop > (aobj->u_pages << PAGE_SHIFT)) {
781: printf("uao_flush: strange, got an out of range "
782: "flush (fixed)\n");
783: stop = aobj->u_pages << PAGE_SHIFT;
784: }
785: by_list = (uobj->uo_npages <=
1.46 chs 786: ((stop - start) >> PAGE_SHIFT) * UVM_PAGE_HASH_PENALTY);
1.22 thorpej 787: }
788: UVMHIST_LOG(maphist,
789: " flush start=0x%lx, stop=0x%x, by_list=%d, flags=0x%x",
790: start, stop, by_list, flags);
1.1 mrg 791:
1.5 mrg 792: /*
1.22 thorpej 793: * Don't need to do any work here if we're not freeing
794: * or deactivating pages.
795: */
1.46 chs 796:
1.22 thorpej 797: if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
1.46 chs 798: simple_unlock(&uobj->vmobjlock);
799: return 0;
1.22 thorpej 800: }
801:
1.5 mrg 802: /*
1.46 chs 803: * now do it. note: we must update nextpg in the body of loop or we
804: * will get stuck. we need to use nextpg because we may free "pg"
1.22 thorpej 805: * before doing the next loop.
1.21 thorpej 806: */
1.22 thorpej 807:
808: if (by_list) {
1.46 chs 809: pg = TAILQ_FIRST(&uobj->memq);
1.22 thorpej 810: } else {
811: curoff = start;
1.46 chs 812: pg = uvm_pagelookup(uobj, curoff);
1.22 thorpej 813: }
814:
1.46 chs 815: nextpg = NULL;
816: uvm_lock_pageq();
1.22 thorpej 817:
818: /* locked: both page queues and uobj */
1.46 chs 819: for ( ; (by_list && pg != NULL) ||
820: (!by_list && curoff < stop) ; pg = nextpg) {
1.22 thorpej 821: if (by_list) {
1.46 chs 822: nextpg = TAILQ_NEXT(pg, listq);
823: if (pg->offset < start || pg->offset >= stop)
1.22 thorpej 824: continue;
825: } else {
826: curoff += PAGE_SIZE;
827: if (curoff < stop)
1.46 chs 828: nextpg = uvm_pagelookup(uobj, curoff);
829: if (pg == NULL)
1.22 thorpej 830: continue;
831: }
1.46 chs 832: switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
1.41 chs 833:
1.22 thorpej 834: /*
835: * XXX In these first 3 cases, we always just
836: * XXX deactivate the page. We may want to
837: * XXX handle the different cases more specifically
838: * XXX in the future.
839: */
1.46 chs 840:
1.22 thorpej 841: case PGO_CLEANIT|PGO_FREE:
842: case PGO_CLEANIT|PGO_DEACTIVATE:
843: case PGO_DEACTIVATE:
1.25 thorpej 844: deactivate_it:
1.22 thorpej 845: /* skip the page if it's loaned or wired */
1.46 chs 846: if (pg->loan_count != 0 || pg->wire_count != 0)
1.22 thorpej 847: continue;
848:
849: /* ...and deactivate the page. */
1.46 chs 850: pmap_clear_reference(pg);
851: uvm_pagedeactivate(pg);
1.22 thorpej 852: continue;
853:
854: case PGO_FREE:
1.46 chs 855:
1.25 thorpej 856: /*
857: * If there are multiple references to
858: * the object, just deactivate the page.
859: */
1.46 chs 860:
1.25 thorpej 861: if (uobj->uo_refs > 1)
862: goto deactivate_it;
863:
1.22 thorpej 864: /* XXX skip the page if it's loaned or wired */
1.46 chs 865: if (pg->loan_count != 0 || pg->wire_count != 0)
1.22 thorpej 866: continue;
867:
868: /*
1.46 chs 869: * wait if the page is busy, then free the swap slot
870: * and the page.
1.22 thorpej 871: */
1.46 chs 872:
873: pmap_page_protect(pg, VM_PROT_NONE);
874: while (pg->flags & PG_BUSY) {
875: pg->flags |= PG_WANTED;
876: uvm_unlock_pageq();
877: UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, 0,
878: "uao_put", 0);
879: simple_lock(&uobj->vmobjlock);
880: uvm_lock_pageq();
1.22 thorpej 881: }
1.46 chs 882: uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
883: uvm_pagefree(pg);
1.22 thorpej 884: continue;
885: }
886: }
887: uvm_unlock_pageq();
1.46 chs 888: simple_unlock(&uobj->vmobjlock);
889: return 0;
1.1 mrg 890: }
891:
892: /*
893: * uao_get: fetch me a page
894: *
895: * we have three cases:
896: * 1: page is resident -> just return the page.
897: * 2: page is zero-fill -> allocate a new page and zero it.
898: * 3: page is swapped out -> fetch the page from swap.
899: *
900: * cases 1 and 2 can be handled with PGO_LOCKED, case 3 cannot.
901: * so, if the "center" page hits case 3 (or any page, with PGO_ALLPAGES),
1.40 chs 902: * then we will need to return EBUSY.
1.1 mrg 903: *
904: * => prefer map unlocked (not required)
905: * => object must be locked! we will _unlock_ it before starting any I/O.
906: * => flags: PGO_ALLPAGES: get all of the pages
907: * PGO_LOCKED: fault data structures are locked
908: * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
909: * => NOTE: caller must check for released pages!!
910: */
1.46 chs 911:
1.5 mrg 912: static int
913: uao_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags)
914: struct uvm_object *uobj;
1.28 kleink 915: voff_t offset;
1.5 mrg 916: struct vm_page **pps;
917: int *npagesp;
918: int centeridx, advice, flags;
919: vm_prot_t access_type;
920: {
921: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.28 kleink 922: voff_t current_offset;
1.43 chs 923: struct vm_page *ptmp;
1.46 chs 924: int lcv, gotpages, maxpages, swslot, error, pageidx;
1.5 mrg 925: boolean_t done;
926: UVMHIST_FUNC("uao_get"); UVMHIST_CALLED(pdhist);
927:
1.27 chs 928: UVMHIST_LOG(pdhist, "aobj=%p offset=%d, flags=%d",
929: aobj, offset, flags,0);
1.37 chs 930:
1.5 mrg 931: /*
932: * get number of pages
933: */
1.46 chs 934:
1.5 mrg 935: maxpages = *npagesp;
936:
937: /*
938: * step 1: handled the case where fault data structures are locked.
939: */
1.1 mrg 940:
1.5 mrg 941: if (flags & PGO_LOCKED) {
1.46 chs 942:
1.5 mrg 943: /*
944: * step 1a: get pages that are already resident. only do
945: * this if the data structures are locked (i.e. the first
946: * time through).
947: */
948:
949: done = TRUE; /* be optimistic */
950: gotpages = 0; /* # of pages we got so far */
951: for (lcv = 0, current_offset = offset ; lcv < maxpages ;
952: lcv++, current_offset += PAGE_SIZE) {
953: /* do we care about this page? if not, skip it */
954: if (pps[lcv] == PGO_DONTCARE)
955: continue;
956: ptmp = uvm_pagelookup(uobj, current_offset);
957:
958: /*
1.30 thorpej 959: * if page is new, attempt to allocate the page,
960: * zero-fill'd.
1.5 mrg 961: */
1.46 chs 962:
963: if (ptmp == NULL && uao_find_swslot(&aobj->u_obj,
1.15 chs 964: current_offset >> PAGE_SHIFT) == 0) {
1.5 mrg 965: ptmp = uvm_pagealloc(uobj, current_offset,
1.30 thorpej 966: NULL, UVM_PGA_ZERO);
1.5 mrg 967: if (ptmp) {
968: /* new page */
1.47 chs 969: ptmp->flags &= ~(PG_FAKE);
1.5 mrg 970: ptmp->pqflags |= PQ_AOBJ;
1.47 chs 971: goto gotpage;
1.5 mrg 972: }
973: }
974:
975: /*
1.46 chs 976: * to be useful must get a non-busy page
1.5 mrg 977: */
1.46 chs 978:
979: if (ptmp == NULL || (ptmp->flags & PG_BUSY) != 0) {
1.5 mrg 980: if (lcv == centeridx ||
981: (flags & PGO_ALLPAGES) != 0)
982: /* need to do a wait or I/O! */
1.41 chs 983: done = FALSE;
1.5 mrg 984: continue;
985: }
986:
987: /*
988: * useful page: busy/lock it and plug it in our
989: * result array
990: */
1.46 chs 991:
1.5 mrg 992: /* caller must un-busy this page */
1.41 chs 993: ptmp->flags |= PG_BUSY;
1.5 mrg 994: UVM_PAGE_OWN(ptmp, "uao_get1");
1.47 chs 995: gotpage:
1.5 mrg 996: pps[lcv] = ptmp;
997: gotpages++;
1.46 chs 998: }
1.5 mrg 999:
1000: /*
1001: * step 1b: now we've either done everything needed or we
1002: * to unlock and do some waiting or I/O.
1003: */
1004:
1005: UVMHIST_LOG(pdhist, "<- done (done=%d)", done, 0,0,0);
1006: *npagesp = gotpages;
1007: if (done)
1.46 chs 1008: return 0;
1.5 mrg 1009: else
1.46 chs 1010: return EBUSY;
1.1 mrg 1011: }
1012:
1.5 mrg 1013: /*
1014: * step 2: get non-resident or busy pages.
1015: * object is locked. data structures are unlocked.
1016: */
1017:
1018: for (lcv = 0, current_offset = offset ; lcv < maxpages ;
1019: lcv++, current_offset += PAGE_SIZE) {
1.27 chs 1020:
1.5 mrg 1021: /*
1022: * - skip over pages we've already gotten or don't want
1023: * - skip over pages we don't _have_ to get
1024: */
1.27 chs 1025:
1.5 mrg 1026: if (pps[lcv] != NULL ||
1027: (lcv != centeridx && (flags & PGO_ALLPAGES) == 0))
1028: continue;
1029:
1.27 chs 1030: pageidx = current_offset >> PAGE_SHIFT;
1031:
1.5 mrg 1032: /*
1033: * we have yet to locate the current page (pps[lcv]). we
1034: * first look for a page that is already at the current offset.
1035: * if we find a page, we check to see if it is busy or
1036: * released. if that is the case, then we sleep on the page
1037: * until it is no longer busy or released and repeat the lookup.
1038: * if the page we found is neither busy nor released, then we
1039: * busy it (so we own it) and plug it into pps[lcv]. this
1040: * 'break's the following while loop and indicates we are
1041: * ready to move on to the next page in the "lcv" loop above.
1042: *
1043: * if we exit the while loop with pps[lcv] still set to NULL,
1044: * then it means that we allocated a new busy/fake/clean page
1045: * ptmp in the object and we need to do I/O to fill in the data.
1046: */
1047:
1048: /* top of "pps" while loop */
1049: while (pps[lcv] == NULL) {
1050: /* look for a resident page */
1051: ptmp = uvm_pagelookup(uobj, current_offset);
1052:
1053: /* not resident? allocate one now (if we can) */
1054: if (ptmp == NULL) {
1055:
1056: ptmp = uvm_pagealloc(uobj, current_offset,
1.19 chs 1057: NULL, 0);
1.5 mrg 1058:
1059: /* out of RAM? */
1060: if (ptmp == NULL) {
1061: simple_unlock(&uobj->vmobjlock);
1062: UVMHIST_LOG(pdhist,
1063: "sleeping, ptmp == NULL\n",0,0,0,0);
1064: uvm_wait("uao_getpage");
1065: simple_lock(&uobj->vmobjlock);
1.41 chs 1066: continue;
1.5 mrg 1067: }
1068:
1069: /*
1070: * safe with PQ's unlocked: because we just
1071: * alloc'd the page
1072: */
1.46 chs 1073:
1.5 mrg 1074: ptmp->pqflags |= PQ_AOBJ;
1075:
1.41 chs 1076: /*
1.5 mrg 1077: * got new page ready for I/O. break pps while
1078: * loop. pps[lcv] is still NULL.
1079: */
1.46 chs 1080:
1.5 mrg 1081: break;
1082: }
1083:
1084: /* page is there, see if we need to wait on it */
1.46 chs 1085: if ((ptmp->flags & PG_BUSY) != 0) {
1.5 mrg 1086: ptmp->flags |= PG_WANTED;
1087: UVMHIST_LOG(pdhist,
1088: "sleeping, ptmp->flags 0x%x\n",
1089: ptmp->flags,0,0,0);
1.23 thorpej 1090: UVM_UNLOCK_AND_WAIT(ptmp, &uobj->vmobjlock,
1091: FALSE, "uao_get", 0);
1.5 mrg 1092: simple_lock(&uobj->vmobjlock);
1.46 chs 1093: continue;
1.5 mrg 1094: }
1.41 chs 1095:
1096: /*
1.5 mrg 1097: * if we get here then the page has become resident and
1098: * unbusy between steps 1 and 2. we busy it now (so we
1099: * own it) and set pps[lcv] (so that we exit the while
1100: * loop).
1101: */
1.46 chs 1102:
1.5 mrg 1103: /* we own it, caller must un-busy */
1104: ptmp->flags |= PG_BUSY;
1105: UVM_PAGE_OWN(ptmp, "uao_get2");
1106: pps[lcv] = ptmp;
1107: }
1108:
1109: /*
1110: * if we own the valid page at the correct offset, pps[lcv] will
1111: * point to it. nothing more to do except go to the next page.
1112: */
1.46 chs 1113:
1.5 mrg 1114: if (pps[lcv])
1115: continue; /* next lcv */
1116:
1117: /*
1.41 chs 1118: * we have a "fake/busy/clean" page that we just allocated.
1.5 mrg 1119: * do the needed "i/o", either reading from swap or zeroing.
1120: */
1.46 chs 1121:
1122: swslot = uao_find_swslot(&aobj->u_obj, pageidx);
1.5 mrg 1123:
1124: /*
1125: * just zero the page if there's nothing in swap.
1126: */
1.46 chs 1127:
1128: if (swslot == 0) {
1129:
1.5 mrg 1130: /*
1131: * page hasn't existed before, just zero it.
1132: */
1.46 chs 1133:
1.5 mrg 1134: uvm_pagezero(ptmp);
1.27 chs 1135: } else {
1.5 mrg 1136: UVMHIST_LOG(pdhist, "pagein from swslot %d",
1137: swslot, 0,0,0);
1138:
1139: /*
1140: * page in the swapped-out page.
1141: * unlock object for i/o, relock when done.
1142: */
1.46 chs 1143:
1.5 mrg 1144: simple_unlock(&uobj->vmobjlock);
1.46 chs 1145: error = uvm_swap_get(ptmp, swslot, PGO_SYNCIO);
1.5 mrg 1146: simple_lock(&uobj->vmobjlock);
1147:
1148: /*
1149: * I/O done. check for errors.
1150: */
1.46 chs 1151:
1152: if (error != 0) {
1.5 mrg 1153: UVMHIST_LOG(pdhist, "<- done (error=%d)",
1.46 chs 1154: error,0,0,0);
1.5 mrg 1155: if (ptmp->flags & PG_WANTED)
1.24 thorpej 1156: wakeup(ptmp);
1.27 chs 1157:
1158: /*
1159: * remove the swap slot from the aobj
1160: * and mark the aobj as having no real slot.
1161: * don't free the swap slot, thus preventing
1162: * it from being used again.
1163: */
1.46 chs 1164:
1.27 chs 1165: swslot = uao_set_swslot(&aobj->u_obj, pageidx,
1166: SWSLOT_BAD);
1.45 chs 1167: if (swslot != -1) {
1168: uvm_swap_markbad(swslot, 1);
1169: }
1.27 chs 1170:
1.5 mrg 1171: uvm_lock_pageq();
1172: uvm_pagefree(ptmp);
1173: uvm_unlock_pageq();
1174: simple_unlock(&uobj->vmobjlock);
1.46 chs 1175: return error;
1.5 mrg 1176: }
1177: }
1178:
1.41 chs 1179: /*
1.5 mrg 1180: * we got the page! clear the fake flag (indicates valid
1181: * data now in page) and plug into our result array. note
1.41 chs 1182: * that page is still busy.
1.5 mrg 1183: *
1184: * it is the callers job to:
1185: * => check if the page is released
1186: * => unbusy the page
1187: * => activate the page
1188: */
1189:
1.46 chs 1190: ptmp->flags &= ~PG_FAKE;
1.5 mrg 1191: pps[lcv] = ptmp;
1.46 chs 1192: }
1.1 mrg 1193:
1194: /*
1.5 mrg 1195: * finally, unlock object and return.
1196: */
1.1 mrg 1197:
1198: simple_unlock(&uobj->vmobjlock);
1.5 mrg 1199: UVMHIST_LOG(pdhist, "<- done (OK)",0,0,0,0);
1.46 chs 1200: return 0;
1.1 mrg 1201: }
1202:
1203: /*
1.18 chs 1204: * uao_dropswap: release any swap resources from this aobj page.
1.41 chs 1205: *
1.18 chs 1206: * => aobj must be locked or have a reference count of 0.
1207: */
1208:
1209: void
1210: uao_dropswap(uobj, pageidx)
1211: struct uvm_object *uobj;
1212: int pageidx;
1213: {
1214: int slot;
1215:
1216: slot = uao_set_swslot(uobj, pageidx, 0);
1217: if (slot) {
1218: uvm_swap_free(slot, 1);
1219: }
1.27 chs 1220: }
1221:
1222: /*
1223: * page in every page in every aobj that is paged-out to a range of swslots.
1.41 chs 1224: *
1.27 chs 1225: * => nothing should be locked.
1226: * => returns TRUE if pagein was aborted due to lack of memory.
1227: */
1.46 chs 1228:
1.27 chs 1229: boolean_t
1230: uao_swap_off(startslot, endslot)
1231: int startslot, endslot;
1232: {
1233: struct uvm_aobj *aobj, *nextaobj;
1.46 chs 1234: boolean_t rv;
1.27 chs 1235:
1236: /*
1237: * walk the list of all aobjs.
1238: */
1239:
1240: restart:
1241: simple_lock(&uao_list_lock);
1242: for (aobj = LIST_FIRST(&uao_list);
1243: aobj != NULL;
1244: aobj = nextaobj) {
1245:
1246: /*
1.46 chs 1247: * try to get the object lock, start all over if we fail.
1.27 chs 1248: * most of the time we'll get the aobj lock,
1249: * so this should be a rare case.
1250: */
1.46 chs 1251:
1.27 chs 1252: if (!simple_lock_try(&aobj->u_obj.vmobjlock)) {
1253: simple_unlock(&uao_list_lock);
1254: goto restart;
1255: }
1256:
1257: /*
1258: * add a ref to the aobj so it doesn't disappear
1259: * while we're working.
1260: */
1.46 chs 1261:
1.27 chs 1262: uao_reference_locked(&aobj->u_obj);
1263:
1264: /*
1265: * now it's safe to unlock the uao list.
1266: */
1.46 chs 1267:
1.27 chs 1268: simple_unlock(&uao_list_lock);
1269:
1270: /*
1271: * page in any pages in the swslot range.
1272: * if there's an error, abort and return the error.
1273: */
1.46 chs 1274:
1.27 chs 1275: rv = uao_pagein(aobj, startslot, endslot);
1276: if (rv) {
1277: uao_detach_locked(&aobj->u_obj);
1278: return rv;
1279: }
1280:
1281: /*
1282: * we're done with this aobj.
1283: * relock the list and drop our ref on the aobj.
1284: */
1.46 chs 1285:
1.27 chs 1286: simple_lock(&uao_list_lock);
1287: nextaobj = LIST_NEXT(aobj, u_list);
1288: uao_detach_locked(&aobj->u_obj);
1289: }
1290:
1291: /*
1292: * done with traversal, unlock the list
1293: */
1294: simple_unlock(&uao_list_lock);
1295: return FALSE;
1296: }
1297:
1298:
1299: /*
1300: * page in any pages from aobj in the given range.
1301: *
1302: * => aobj must be locked and is returned locked.
1303: * => returns TRUE if pagein was aborted due to lack of memory.
1304: */
1305: static boolean_t
1306: uao_pagein(aobj, startslot, endslot)
1307: struct uvm_aobj *aobj;
1308: int startslot, endslot;
1309: {
1310: boolean_t rv;
1311:
1312: if (UAO_USES_SWHASH(aobj)) {
1313: struct uao_swhash_elt *elt;
1314: int bucket;
1315:
1316: restart:
1317: for (bucket = aobj->u_swhashmask; bucket >= 0; bucket--) {
1318: for (elt = LIST_FIRST(&aobj->u_swhash[bucket]);
1319: elt != NULL;
1320: elt = LIST_NEXT(elt, list)) {
1321: int i;
1322:
1323: for (i = 0; i < UAO_SWHASH_CLUSTER_SIZE; i++) {
1324: int slot = elt->slots[i];
1325:
1326: /*
1327: * if the slot isn't in range, skip it.
1328: */
1.46 chs 1329:
1.41 chs 1330: if (slot < startslot ||
1.27 chs 1331: slot >= endslot) {
1332: continue;
1333: }
1334:
1335: /*
1336: * process the page,
1337: * the start over on this object
1338: * since the swhash elt
1339: * may have been freed.
1340: */
1.46 chs 1341:
1.27 chs 1342: rv = uao_pagein_page(aobj,
1343: UAO_SWHASH_ELT_PAGEIDX_BASE(elt) + i);
1344: if (rv) {
1345: return rv;
1346: }
1347: goto restart;
1348: }
1349: }
1350: }
1351: } else {
1352: int i;
1353:
1354: for (i = 0; i < aobj->u_pages; i++) {
1355: int slot = aobj->u_swslots[i];
1356:
1357: /*
1358: * if the slot isn't in range, skip it
1359: */
1.46 chs 1360:
1.27 chs 1361: if (slot < startslot || slot >= endslot) {
1362: continue;
1363: }
1364:
1365: /*
1366: * process the page.
1367: */
1.46 chs 1368:
1.27 chs 1369: rv = uao_pagein_page(aobj, i);
1370: if (rv) {
1371: return rv;
1372: }
1373: }
1374: }
1375:
1376: return FALSE;
1377: }
1378:
1379: /*
1380: * page in a page from an aobj. used for swap_off.
1381: * returns TRUE if pagein was aborted due to lack of memory.
1382: *
1383: * => aobj must be locked and is returned locked.
1384: */
1.46 chs 1385:
1.27 chs 1386: static boolean_t
1387: uao_pagein_page(aobj, pageidx)
1388: struct uvm_aobj *aobj;
1389: int pageidx;
1390: {
1391: struct vm_page *pg;
1392: int rv, slot, npages;
1393:
1394: pg = NULL;
1395: npages = 1;
1396: /* locked: aobj */
1397: rv = uao_get(&aobj->u_obj, pageidx << PAGE_SHIFT,
1398: &pg, &npages, 0, VM_PROT_READ|VM_PROT_WRITE, 0, 0);
1399: /* unlocked: aobj */
1400:
1401: /*
1402: * relock and finish up.
1403: */
1.46 chs 1404:
1.27 chs 1405: simple_lock(&aobj->u_obj.vmobjlock);
1406: switch (rv) {
1.40 chs 1407: case 0:
1.27 chs 1408: break;
1409:
1.40 chs 1410: case EIO:
1411: case ERESTART:
1.46 chs 1412:
1.27 chs 1413: /*
1414: * nothing more to do on errors.
1.40 chs 1415: * ERESTART can only mean that the anon was freed,
1.27 chs 1416: * so again there's nothing to do.
1417: */
1.46 chs 1418:
1.27 chs 1419: return FALSE;
1420: }
1421:
1422: /*
1423: * ok, we've got the page now.
1424: * mark it as dirty, clear its swslot and un-busy it.
1425: */
1.46 chs 1426:
1.27 chs 1427: slot = uao_set_swslot(&aobj->u_obj, pageidx, 0);
1428: uvm_swap_free(slot, 1);
1429: pg->flags &= ~(PG_BUSY|PG_CLEAN|PG_FAKE);
1430: UVM_PAGE_OWN(pg, NULL);
1431:
1432: /*
1.46 chs 1433: * deactivate the page (to make sure it's on a page queue).
1.27 chs 1434: */
1.46 chs 1435:
1.27 chs 1436: uvm_lock_pageq();
1437: uvm_pagedeactivate(pg);
1438: uvm_unlock_pageq();
1439: return FALSE;
1.1 mrg 1440: }
CVSweb <webmaster@jp.NetBSD.org>