Annotation of src/sys/uvm/uvm_aobj.c, Revision 1.55
1.55 ! pk 1: /* $NetBSD: uvm_aobj.c,v 1.54 2003/02/01 06:23:54 thorpej Exp $ */
1.6 mrg 2:
1.7 chs 3: /*
4: * Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
5: * Washington University.
6: * All rights reserved.
7: *
8: * Redistribution and use in source and binary forms, with or without
9: * modification, are permitted provided that the following conditions
10: * are met:
11: * 1. Redistributions of source code must retain the above copyright
12: * notice, this list of conditions and the following disclaimer.
13: * 2. Redistributions in binary form must reproduce the above copyright
14: * notice, this list of conditions and the following disclaimer in the
15: * documentation and/or other materials provided with the distribution.
16: * 3. All advertising materials mentioning features or use of this software
17: * must display the following acknowledgement:
18: * This product includes software developed by Charles D. Cranor and
19: * Washington University.
20: * 4. The name of the author may not be used to endorse or promote products
21: * derived from this software without specific prior written permission.
22: *
23: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33: *
1.4 mrg 34: * from: Id: uvm_aobj.c,v 1.1.2.5 1998/02/06 05:14:38 chs Exp
35: */
1.7 chs 36: /*
37: * uvm_aobj.c: anonymous memory uvm_object pager
38: *
39: * author: Chuck Silvers <chuq@chuq.com>
40: * started: Jan-1998
41: *
42: * - design mostly from Chuck Cranor
43: */
1.49 lukem 44:
45: #include <sys/cdefs.h>
1.55 ! pk 46: __KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.54 2003/02/01 06:23:54 thorpej Exp $");
1.7 chs 47:
48: #include "opt_uvmhist.h"
1.1 mrg 49:
50: #include <sys/param.h>
51: #include <sys/systm.h>
52: #include <sys/proc.h>
53: #include <sys/malloc.h>
1.37 chs 54: #include <sys/kernel.h>
1.12 thorpej 55: #include <sys/pool.h>
1.27 chs 56: #include <sys/kernel.h>
1.1 mrg 57:
58: #include <uvm/uvm.h>
59:
60: /*
61: * an aobj manages anonymous-memory backed uvm_objects. in addition
62: * to keeping the list of resident pages, it also keeps a list of
63: * allocated swap blocks. depending on the size of the aobj this list
64: * of allocated swap blocks is either stored in an array (small objects)
65: * or in a hash table (large objects).
66: */
67:
68: /*
69: * local structures
70: */
71:
72: /*
73: * for hash tables, we break the address space of the aobj into blocks
74: * of UAO_SWHASH_CLUSTER_SIZE pages. we require the cluster size to
75: * be a power of two.
76: */
77:
78: #define UAO_SWHASH_CLUSTER_SHIFT 4
79: #define UAO_SWHASH_CLUSTER_SIZE (1 << UAO_SWHASH_CLUSTER_SHIFT)
80:
81: /* get the "tag" for this page index */
82: #define UAO_SWHASH_ELT_TAG(PAGEIDX) \
83: ((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT)
84:
85: /* given an ELT and a page index, find the swap slot */
86: #define UAO_SWHASH_ELT_PAGESLOT(ELT, PAGEIDX) \
87: ((ELT)->slots[(PAGEIDX) & (UAO_SWHASH_CLUSTER_SIZE - 1)])
88:
89: /* given an ELT, return its pageidx base */
90: #define UAO_SWHASH_ELT_PAGEIDX_BASE(ELT) \
91: ((ELT)->tag << UAO_SWHASH_CLUSTER_SHIFT)
92:
93: /*
94: * the swhash hash function
95: */
1.46 chs 96:
1.1 mrg 97: #define UAO_SWHASH_HASH(AOBJ, PAGEIDX) \
98: (&(AOBJ)->u_swhash[(((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT) \
99: & (AOBJ)->u_swhashmask)])
100:
101: /*
102: * the swhash threshhold determines if we will use an array or a
103: * hash table to store the list of allocated swap blocks.
104: */
105:
106: #define UAO_SWHASH_THRESHOLD (UAO_SWHASH_CLUSTER_SIZE * 4)
107: #define UAO_USES_SWHASH(AOBJ) \
108: ((AOBJ)->u_pages > UAO_SWHASH_THRESHOLD) /* use hash? */
109:
110: /*
1.3 chs 111: * the number of buckets in a swhash, with an upper bound
1.1 mrg 112: */
1.46 chs 113:
1.1 mrg 114: #define UAO_SWHASH_MAXBUCKETS 256
115: #define UAO_SWHASH_BUCKETS(AOBJ) \
1.46 chs 116: (MIN((AOBJ)->u_pages >> UAO_SWHASH_CLUSTER_SHIFT, \
1.1 mrg 117: UAO_SWHASH_MAXBUCKETS))
118:
119:
120: /*
121: * uao_swhash_elt: when a hash table is being used, this structure defines
122: * the format of an entry in the bucket list.
123: */
124:
125: struct uao_swhash_elt {
1.5 mrg 126: LIST_ENTRY(uao_swhash_elt) list; /* the hash list */
1.28 kleink 127: voff_t tag; /* our 'tag' */
1.5 mrg 128: int count; /* our number of active slots */
129: int slots[UAO_SWHASH_CLUSTER_SIZE]; /* the slots */
1.1 mrg 130: };
131:
132: /*
133: * uao_swhash: the swap hash table structure
134: */
135:
136: LIST_HEAD(uao_swhash, uao_swhash_elt);
137:
1.12 thorpej 138: /*
139: * uao_swhash_elt_pool: pool of uao_swhash_elt structures
140: */
141:
142: struct pool uao_swhash_elt_pool;
1.1 mrg 143:
144: /*
145: * uvm_aobj: the actual anon-backed uvm_object
146: *
147: * => the uvm_object is at the top of the structure, this allows
1.46 chs 148: * (struct uvm_aobj *) == (struct uvm_object *)
1.1 mrg 149: * => only one of u_swslots and u_swhash is used in any given aobj
150: */
151:
152: struct uvm_aobj {
1.5 mrg 153: struct uvm_object u_obj; /* has: lock, pgops, memq, #pages, #refs */
1.11 drochner 154: int u_pages; /* number of pages in entire object */
1.5 mrg 155: int u_flags; /* the flags (see uvm_aobj.h) */
156: int *u_swslots; /* array of offset->swapslot mappings */
157: /*
158: * hashtable of offset->swapslot mappings
159: * (u_swhash is an array of bucket heads)
160: */
161: struct uao_swhash *u_swhash;
162: u_long u_swhashmask; /* mask for hashtable */
163: LIST_ENTRY(uvm_aobj) u_list; /* global list of aobjs */
1.1 mrg 164: };
165:
166: /*
1.12 thorpej 167: * uvm_aobj_pool: pool of uvm_aobj structures
168: */
169:
170: struct pool uvm_aobj_pool;
1.54 thorpej 171:
172: MALLOC_DEFINE(M_UVMAOBJ, "UVM aobj", "UVM aobj and related structures");
1.12 thorpej 173:
174: /*
1.1 mrg 175: * local functions
176: */
177:
1.46 chs 178: static struct uao_swhash_elt *uao_find_swhash_elt
179: __P((struct uvm_aobj *, int, boolean_t));
180:
181: static void uao_free __P((struct uvm_aobj *));
182: static int uao_get __P((struct uvm_object *, voff_t, struct vm_page **,
183: int *, int, vm_prot_t, int, int));
184: static boolean_t uao_put __P((struct uvm_object *, voff_t, voff_t, int));
185: static boolean_t uao_pagein __P((struct uvm_aobj *, int, int));
186: static boolean_t uao_pagein_page __P((struct uvm_aobj *, int));
1.1 mrg 187:
188: /*
189: * aobj_pager
1.41 chs 190: *
1.1 mrg 191: * note that some functions (e.g. put) are handled elsewhere
192: */
193:
194: struct uvm_pagerops aobj_pager = {
1.27 chs 195: NULL, /* init */
1.5 mrg 196: uao_reference, /* reference */
197: uao_detach, /* detach */
198: NULL, /* fault */
199: uao_get, /* get */
1.46 chs 200: uao_put, /* flush */
1.1 mrg 201: };
202:
203: /*
204: * uao_list: global list of active aobjs, locked by uao_list_lock
205: */
206:
207: static LIST_HEAD(aobjlist, uvm_aobj) uao_list;
1.42 chs 208: static struct simplelock uao_list_lock;
1.1 mrg 209:
210: /*
211: * functions
212: */
213:
214: /*
215: * hash table/array related functions
216: */
217:
218: /*
219: * uao_find_swhash_elt: find (or create) a hash table entry for a page
220: * offset.
221: *
222: * => the object should be locked by the caller
223: */
224:
1.5 mrg 225: static struct uao_swhash_elt *
226: uao_find_swhash_elt(aobj, pageidx, create)
227: struct uvm_aobj *aobj;
228: int pageidx;
229: boolean_t create;
230: {
231: struct uao_swhash *swhash;
232: struct uao_swhash_elt *elt;
1.28 kleink 233: voff_t page_tag;
1.1 mrg 234:
1.45 chs 235: swhash = UAO_SWHASH_HASH(aobj, pageidx);
236: page_tag = UAO_SWHASH_ELT_TAG(pageidx);
1.1 mrg 237:
1.5 mrg 238: /*
239: * now search the bucket for the requested tag
240: */
1.45 chs 241:
1.37 chs 242: LIST_FOREACH(elt, swhash, list) {
1.45 chs 243: if (elt->tag == page_tag) {
244: return elt;
245: }
1.5 mrg 246: }
1.45 chs 247: if (!create) {
1.5 mrg 248: return NULL;
1.45 chs 249: }
1.5 mrg 250:
251: /*
1.12 thorpej 252: * allocate a new entry for the bucket and init/insert it in
1.5 mrg 253: */
1.45 chs 254:
255: elt = pool_get(&uao_swhash_elt_pool, PR_NOWAIT);
256: if (elt == NULL) {
257: return NULL;
258: }
1.5 mrg 259: LIST_INSERT_HEAD(swhash, elt, list);
260: elt->tag = page_tag;
261: elt->count = 0;
1.9 perry 262: memset(elt->slots, 0, sizeof(elt->slots));
1.45 chs 263: return elt;
1.1 mrg 264: }
265:
266: /*
267: * uao_find_swslot: find the swap slot number for an aobj/pageidx
268: *
1.41 chs 269: * => object must be locked by caller
1.1 mrg 270: */
1.46 chs 271:
272: int
273: uao_find_swslot(uobj, pageidx)
274: struct uvm_object *uobj;
1.11 drochner 275: int pageidx;
1.1 mrg 276: {
1.46 chs 277: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
278: struct uao_swhash_elt *elt;
1.1 mrg 279:
1.5 mrg 280: /*
281: * if noswap flag is set, then we never return a slot
282: */
1.1 mrg 283:
1.5 mrg 284: if (aobj->u_flags & UAO_FLAG_NOSWAP)
285: return(0);
1.1 mrg 286:
1.5 mrg 287: /*
288: * if hashing, look in hash table.
289: */
1.1 mrg 290:
1.5 mrg 291: if (UAO_USES_SWHASH(aobj)) {
1.46 chs 292: elt = uao_find_swhash_elt(aobj, pageidx, FALSE);
1.5 mrg 293: if (elt)
294: return(UAO_SWHASH_ELT_PAGESLOT(elt, pageidx));
295: else
1.31 thorpej 296: return(0);
1.5 mrg 297: }
1.1 mrg 298:
1.41 chs 299: /*
1.5 mrg 300: * otherwise, look in the array
301: */
1.46 chs 302:
1.5 mrg 303: return(aobj->u_swslots[pageidx]);
1.1 mrg 304: }
305:
306: /*
307: * uao_set_swslot: set the swap slot for a page in an aobj.
308: *
309: * => setting a slot to zero frees the slot
310: * => object must be locked by caller
1.45 chs 311: * => we return the old slot number, or -1 if we failed to allocate
312: * memory to record the new slot number
1.1 mrg 313: */
1.46 chs 314:
1.5 mrg 315: int
316: uao_set_swslot(uobj, pageidx, slot)
317: struct uvm_object *uobj;
318: int pageidx, slot;
319: {
320: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.45 chs 321: struct uao_swhash_elt *elt;
1.5 mrg 322: int oldslot;
323: UVMHIST_FUNC("uao_set_swslot"); UVMHIST_CALLED(pdhist);
324: UVMHIST_LOG(pdhist, "aobj %p pageidx %d slot %d",
325: aobj, pageidx, slot, 0);
1.1 mrg 326:
1.5 mrg 327: /*
1.46 chs 328: * if noswap flag is set, then we can't set a non-zero slot.
1.5 mrg 329: */
1.1 mrg 330:
1.5 mrg 331: if (aobj->u_flags & UAO_FLAG_NOSWAP) {
332: if (slot == 0)
1.46 chs 333: return(0);
1.1 mrg 334:
1.5 mrg 335: printf("uao_set_swslot: uobj = %p\n", uobj);
1.46 chs 336: panic("uao_set_swslot: NOSWAP object");
1.5 mrg 337: }
1.1 mrg 338:
1.5 mrg 339: /*
340: * are we using a hash table? if so, add it in the hash.
341: */
1.1 mrg 342:
1.5 mrg 343: if (UAO_USES_SWHASH(aobj)) {
1.39 chs 344:
1.12 thorpej 345: /*
346: * Avoid allocating an entry just to free it again if
347: * the page had not swap slot in the first place, and
348: * we are freeing.
349: */
1.39 chs 350:
1.46 chs 351: elt = uao_find_swhash_elt(aobj, pageidx, slot != 0);
1.12 thorpej 352: if (elt == NULL) {
1.45 chs 353: return slot ? -1 : 0;
1.12 thorpej 354: }
1.5 mrg 355:
356: oldslot = UAO_SWHASH_ELT_PAGESLOT(elt, pageidx);
357: UAO_SWHASH_ELT_PAGESLOT(elt, pageidx) = slot;
358:
359: /*
360: * now adjust the elt's reference counter and free it if we've
361: * dropped it to zero.
362: */
363:
364: if (slot) {
365: if (oldslot == 0)
366: elt->count++;
1.45 chs 367: } else {
368: if (oldslot)
1.5 mrg 369: elt->count--;
370:
371: if (elt->count == 0) {
372: LIST_REMOVE(elt, list);
1.12 thorpej 373: pool_put(&uao_swhash_elt_pool, elt);
1.5 mrg 374: }
375: }
1.41 chs 376: } else {
1.5 mrg 377: /* we are using an array */
378: oldslot = aobj->u_swslots[pageidx];
379: aobj->u_swslots[pageidx] = slot;
380: }
381: return (oldslot);
1.1 mrg 382: }
383:
384: /*
385: * end of hash/array functions
386: */
387:
388: /*
389: * uao_free: free all resources held by an aobj, and then free the aobj
390: *
391: * => the aobj should be dead
392: */
1.46 chs 393:
1.1 mrg 394: static void
395: uao_free(aobj)
1.5 mrg 396: struct uvm_aobj *aobj;
1.1 mrg 397: {
1.46 chs 398: int swpgonlydelta = 0;
1.1 mrg 399:
1.27 chs 400: simple_unlock(&aobj->u_obj.vmobjlock);
1.5 mrg 401: if (UAO_USES_SWHASH(aobj)) {
402: int i, hashbuckets = aobj->u_swhashmask + 1;
1.1 mrg 403:
1.5 mrg 404: /*
405: * free the swslots from each hash bucket,
406: * then the hash bucket, and finally the hash table itself.
407: */
1.46 chs 408:
1.5 mrg 409: for (i = 0; i < hashbuckets; i++) {
410: struct uao_swhash_elt *elt, *next;
411:
1.27 chs 412: for (elt = LIST_FIRST(&aobj->u_swhash[i]);
413: elt != NULL;
414: elt = next) {
1.5 mrg 415: int j;
416:
1.27 chs 417: for (j = 0; j < UAO_SWHASH_CLUSTER_SIZE; j++) {
1.5 mrg 418: int slot = elt->slots[j];
419:
1.37 chs 420: if (slot == 0) {
421: continue;
422: }
423: uvm_swap_free(slot, 1);
1.46 chs 424: swpgonlydelta++;
1.5 mrg 425: }
426:
1.27 chs 427: next = LIST_NEXT(elt, list);
1.12 thorpej 428: pool_put(&uao_swhash_elt_pool, elt);
1.5 mrg 429: }
430: }
1.34 thorpej 431: free(aobj->u_swhash, M_UVMAOBJ);
1.5 mrg 432: } else {
433: int i;
434:
435: /*
436: * free the array
437: */
438:
1.27 chs 439: for (i = 0; i < aobj->u_pages; i++) {
1.5 mrg 440: int slot = aobj->u_swslots[i];
441:
1.18 chs 442: if (slot) {
1.5 mrg 443: uvm_swap_free(slot, 1);
1.46 chs 444: swpgonlydelta++;
1.18 chs 445: }
1.5 mrg 446: }
1.34 thorpej 447: free(aobj->u_swslots, M_UVMAOBJ);
1.1 mrg 448: }
449:
1.5 mrg 450: /*
451: * finally free the aobj itself
452: */
1.46 chs 453:
1.12 thorpej 454: pool_put(&uvm_aobj_pool, aobj);
1.46 chs 455:
456: /*
457: * adjust the counter of pages only in swap for all
458: * the swap slots we've freed.
459: */
460:
1.48 chs 461: if (swpgonlydelta > 0) {
462: simple_lock(&uvm.swap_data_lock);
463: KASSERT(uvmexp.swpgonly >= swpgonlydelta);
464: uvmexp.swpgonly -= swpgonlydelta;
465: simple_unlock(&uvm.swap_data_lock);
466: }
1.1 mrg 467: }
468:
469: /*
470: * pager functions
471: */
472:
473: /*
474: * uao_create: create an aobj of the given size and return its uvm_object.
475: *
476: * => for normal use, flags are always zero
477: * => for the kernel object, the flags are:
478: * UAO_FLAG_KERNOBJ - allocate the kernel object (can only happen once)
479: * UAO_FLAG_KERNSWAP - enable swapping of kernel object (" ")
480: */
1.46 chs 481:
1.5 mrg 482: struct uvm_object *
483: uao_create(size, flags)
1.10 eeh 484: vsize_t size;
1.5 mrg 485: int flags;
486: {
1.46 chs 487: static struct uvm_aobj kernel_object_store;
488: static int kobj_alloced = 0;
1.15 chs 489: int pages = round_page(size) >> PAGE_SHIFT;
1.5 mrg 490: struct uvm_aobj *aobj;
1.1 mrg 491:
1.5 mrg 492: /*
1.27 chs 493: * malloc a new aobj unless we are asked for the kernel object
494: */
1.5 mrg 495:
1.46 chs 496: if (flags & UAO_FLAG_KERNOBJ) {
497: KASSERT(!kobj_alloced);
1.5 mrg 498: aobj = &kernel_object_store;
499: aobj->u_pages = pages;
1.46 chs 500: aobj->u_flags = UAO_FLAG_NOSWAP;
1.5 mrg 501: aobj->u_obj.uo_refs = UVM_OBJ_KERN;
502: kobj_alloced = UAO_FLAG_KERNOBJ;
503: } else if (flags & UAO_FLAG_KERNSWAP) {
1.46 chs 504: KASSERT(kobj_alloced == UAO_FLAG_KERNOBJ);
1.5 mrg 505: aobj = &kernel_object_store;
506: kobj_alloced = UAO_FLAG_KERNSWAP;
1.46 chs 507: } else {
1.12 thorpej 508: aobj = pool_get(&uvm_aobj_pool, PR_WAITOK);
1.5 mrg 509: aobj->u_pages = pages;
1.46 chs 510: aobj->u_flags = 0;
511: aobj->u_obj.uo_refs = 1;
1.5 mrg 512: }
1.1 mrg 513:
1.5 mrg 514: /*
515: * allocate hash/array if necessary
516: *
517: * note: in the KERNSWAP case no need to worry about locking since
518: * we are still booting we should be the only thread around.
519: */
1.46 chs 520:
1.5 mrg 521: if (flags == 0 || (flags & UAO_FLAG_KERNSWAP) != 0) {
522: int mflags = (flags & UAO_FLAG_KERNSWAP) != 0 ?
523: M_NOWAIT : M_WAITOK;
524:
525: /* allocate hash table or array depending on object size */
1.27 chs 526: if (UAO_USES_SWHASH(aobj)) {
1.5 mrg 527: aobj->u_swhash = hashinit(UAO_SWHASH_BUCKETS(aobj),
1.35 ad 528: HASH_LIST, M_UVMAOBJ, mflags, &aobj->u_swhashmask);
1.5 mrg 529: if (aobj->u_swhash == NULL)
530: panic("uao_create: hashinit swhash failed");
531: } else {
1.34 thorpej 532: aobj->u_swslots = malloc(pages * sizeof(int),
1.5 mrg 533: M_UVMAOBJ, mflags);
534: if (aobj->u_swslots == NULL)
535: panic("uao_create: malloc swslots failed");
1.9 perry 536: memset(aobj->u_swslots, 0, pages * sizeof(int));
1.5 mrg 537: }
538:
539: if (flags) {
540: aobj->u_flags &= ~UAO_FLAG_NOSWAP; /* clear noswap */
541: return(&aobj->u_obj);
542: }
543: }
544:
545: /*
546: * init aobj fields
547: */
1.46 chs 548:
1.5 mrg 549: simple_lock_init(&aobj->u_obj.vmobjlock);
550: aobj->u_obj.pgops = &aobj_pager;
551: TAILQ_INIT(&aobj->u_obj.memq);
552: aobj->u_obj.uo_npages = 0;
1.1 mrg 553:
1.5 mrg 554: /*
555: * now that aobj is ready, add it to the global list
556: */
1.46 chs 557:
1.5 mrg 558: simple_lock(&uao_list_lock);
559: LIST_INSERT_HEAD(&uao_list, aobj, u_list);
560: simple_unlock(&uao_list_lock);
561: return(&aobj->u_obj);
1.1 mrg 562: }
563:
564:
565:
566: /*
567: * uao_init: set up aobj pager subsystem
568: *
569: * => called at boot time from uvm_pager_init()
570: */
1.46 chs 571:
1.27 chs 572: void
1.46 chs 573: uao_init(void)
1.5 mrg 574: {
1.12 thorpej 575: static int uao_initialized;
576:
577: if (uao_initialized)
578: return;
579: uao_initialized = TRUE;
1.5 mrg 580: LIST_INIT(&uao_list);
581: simple_lock_init(&uao_list_lock);
1.12 thorpej 582:
1.14 thorpej 583: /*
584: * NOTE: Pages fror this pool must not come from a pageable
585: * kernel map!
586: */
1.46 chs 587:
1.12 thorpej 588: pool_init(&uao_swhash_elt_pool, sizeof(struct uao_swhash_elt),
1.50 thorpej 589: 0, 0, 0, "uaoeltpl", NULL);
1.12 thorpej 590: pool_init(&uvm_aobj_pool, sizeof(struct uvm_aobj), 0, 0, 0,
1.50 thorpej 591: "aobjpl", &pool_allocator_nointr);
1.1 mrg 592: }
593:
594: /*
595: * uao_reference: add a ref to an aobj
596: *
1.27 chs 597: * => aobj must be unlocked
598: * => just lock it and call the locked version
1.1 mrg 599: */
1.46 chs 600:
1.5 mrg 601: void
602: uao_reference(uobj)
603: struct uvm_object *uobj;
1.1 mrg 604: {
1.27 chs 605: simple_lock(&uobj->vmobjlock);
606: uao_reference_locked(uobj);
607: simple_unlock(&uobj->vmobjlock);
608: }
609:
610: /*
611: * uao_reference_locked: add a ref to an aobj that is already locked
612: *
613: * => aobj must be locked
614: * this needs to be separate from the normal routine
615: * since sometimes we need to add a reference to an aobj when
616: * it's already locked.
617: */
1.46 chs 618:
1.27 chs 619: void
620: uao_reference_locked(uobj)
621: struct uvm_object *uobj;
622: {
1.5 mrg 623: UVMHIST_FUNC("uao_reference"); UVMHIST_CALLED(maphist);
1.1 mrg 624:
1.5 mrg 625: /*
626: * kernel_object already has plenty of references, leave it alone.
627: */
1.1 mrg 628:
1.20 thorpej 629: if (UVM_OBJ_IS_KERN_OBJECT(uobj))
1.5 mrg 630: return;
1.1 mrg 631:
1.46 chs 632: uobj->uo_refs++;
1.41 chs 633: UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)",
1.27 chs 634: uobj, uobj->uo_refs,0,0);
1.1 mrg 635: }
636:
637: /*
638: * uao_detach: drop a reference to an aobj
639: *
1.27 chs 640: * => aobj must be unlocked
641: * => just lock it and call the locked version
1.1 mrg 642: */
1.46 chs 643:
1.5 mrg 644: void
645: uao_detach(uobj)
646: struct uvm_object *uobj;
647: {
1.27 chs 648: simple_lock(&uobj->vmobjlock);
649: uao_detach_locked(uobj);
650: }
651:
652: /*
653: * uao_detach_locked: drop a reference to an aobj
654: *
655: * => aobj must be locked, and is unlocked (or freed) upon return.
656: * this needs to be separate from the normal routine
657: * since sometimes we need to detach from an aobj when
658: * it's already locked.
659: */
1.46 chs 660:
1.27 chs 661: void
662: uao_detach_locked(uobj)
663: struct uvm_object *uobj;
664: {
1.5 mrg 665: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.46 chs 666: struct vm_page *pg;
1.5 mrg 667: UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist);
1.1 mrg 668:
1.5 mrg 669: /*
670: * detaching from kernel_object is a noop.
671: */
1.46 chs 672:
1.27 chs 673: if (UVM_OBJ_IS_KERN_OBJECT(uobj)) {
674: simple_unlock(&uobj->vmobjlock);
1.5 mrg 675: return;
1.27 chs 676: }
1.5 mrg 677:
678: UVMHIST_LOG(maphist," (uobj=0x%x) ref=%d", uobj,uobj->uo_refs,0,0);
1.46 chs 679: uobj->uo_refs--;
680: if (uobj->uo_refs) {
1.5 mrg 681: simple_unlock(&uobj->vmobjlock);
682: UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
683: return;
684: }
685:
686: /*
687: * remove the aobj from the global list.
688: */
1.46 chs 689:
1.5 mrg 690: simple_lock(&uao_list_lock);
691: LIST_REMOVE(aobj, u_list);
692: simple_unlock(&uao_list_lock);
693:
694: /*
1.46 chs 695: * free all the pages left in the aobj. for each page,
696: * when the page is no longer busy (and thus after any disk i/o that
697: * it's involved in is complete), release any swap resources and
698: * free the page itself.
1.5 mrg 699: */
1.46 chs 700:
701: uvm_lock_pageq();
702: while ((pg = TAILQ_FIRST(&uobj->memq)) != NULL) {
703: pmap_page_protect(pg, VM_PROT_NONE);
1.5 mrg 704: if (pg->flags & PG_BUSY) {
1.46 chs 705: pg->flags |= PG_WANTED;
706: uvm_unlock_pageq();
707: UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, FALSE,
708: "uao_det", 0);
709: simple_lock(&uobj->vmobjlock);
710: uvm_lock_pageq();
1.5 mrg 711: continue;
712: }
1.18 chs 713: uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
1.5 mrg 714: uvm_pagefree(pg);
715: }
1.46 chs 716: uvm_unlock_pageq();
1.1 mrg 717:
1.5 mrg 718: /*
1.46 chs 719: * finally, free the aobj itself.
1.5 mrg 720: */
1.1 mrg 721:
1.5 mrg 722: uao_free(aobj);
723: }
1.1 mrg 724:
725: /*
1.46 chs 726: * uao_put: flush pages out of a uvm object
1.22 thorpej 727: *
728: * => object should be locked by caller. we may _unlock_ the object
729: * if (and only if) we need to clean a page (PGO_CLEANIT).
730: * XXXJRT Currently, however, we don't. In the case of cleaning
731: * XXXJRT a page, we simply just deactivate it. Should probably
732: * XXXJRT handle this better, in the future (although "flushing"
733: * XXXJRT anonymous memory isn't terribly important).
734: * => if PGO_CLEANIT is not set, then we will neither unlock the object
735: * or block.
736: * => if PGO_ALLPAGE is set, then all pages in the object are valid targets
737: * for flushing.
738: * => NOTE: we rely on the fact that the object's memq is a TAILQ and
739: * that new pages are inserted on the tail end of the list. thus,
740: * we can make a complete pass through the object in one go by starting
741: * at the head and working towards the tail (new pages are put in
742: * front of us).
743: * => NOTE: we are allowed to lock the page queues, so the caller
744: * must not be holding the lock on them [e.g. pagedaemon had
745: * better not call us with the queues locked]
746: * => we return TRUE unless we encountered some sort of I/O error
747: * XXXJRT currently never happens, as we never directly initiate
748: * XXXJRT I/O
749: *
750: * note on page traversal:
751: * we can traverse the pages in an object either by going down the
752: * linked list in "uobj->memq", or we can go over the address range
753: * by page doing hash table lookups for each address. depending
754: * on how many pages are in the object it may be cheaper to do one
755: * or the other. we set "by_list" to true if we are using memq.
756: * if the cost of a hash lookup was equal to the cost of the list
757: * traversal we could compare the number of pages in the start->stop
758: * range to the total number of pages in the object. however, it
759: * seems that a hash table lookup is more expensive than the linked
760: * list traversal, so we multiply the number of pages in the
761: * start->stop range by a penalty which we define below.
1.1 mrg 762: */
1.22 thorpej 763:
1.46 chs 764: int
765: uao_put(uobj, start, stop, flags)
1.5 mrg 766: struct uvm_object *uobj;
1.28 kleink 767: voff_t start, stop;
1.5 mrg 768: int flags;
769: {
1.46 chs 770: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.51 enami 771: struct vm_page *pg, *nextpg, curmp, endmp;
1.46 chs 772: boolean_t by_list;
1.28 kleink 773: voff_t curoff;
1.46 chs 774: UVMHIST_FUNC("uao_put"); UVMHIST_CALLED(maphist);
1.22 thorpej 775:
1.46 chs 776: curoff = 0;
1.22 thorpej 777: if (flags & PGO_ALLPAGES) {
778: start = 0;
779: stop = aobj->u_pages << PAGE_SHIFT;
780: by_list = TRUE; /* always go by the list */
781: } else {
782: start = trunc_page(start);
783: stop = round_page(stop);
784: if (stop > (aobj->u_pages << PAGE_SHIFT)) {
785: printf("uao_flush: strange, got an out of range "
786: "flush (fixed)\n");
787: stop = aobj->u_pages << PAGE_SHIFT;
788: }
789: by_list = (uobj->uo_npages <=
1.46 chs 790: ((stop - start) >> PAGE_SHIFT) * UVM_PAGE_HASH_PENALTY);
1.22 thorpej 791: }
792: UVMHIST_LOG(maphist,
793: " flush start=0x%lx, stop=0x%x, by_list=%d, flags=0x%x",
794: start, stop, by_list, flags);
1.1 mrg 795:
1.5 mrg 796: /*
1.22 thorpej 797: * Don't need to do any work here if we're not freeing
798: * or deactivating pages.
799: */
1.46 chs 800:
1.22 thorpej 801: if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
1.46 chs 802: simple_unlock(&uobj->vmobjlock);
803: return 0;
1.22 thorpej 804: }
805:
1.5 mrg 806: /*
1.51 enami 807: * Initialize the marker pages. See the comment in
808: * genfs_putpages() also.
809: */
810:
811: curmp.uobject = uobj;
812: curmp.offset = (voff_t)-1;
813: curmp.flags = PG_BUSY;
814: endmp.uobject = uobj;
815: endmp.offset = (voff_t)-1;
816: endmp.flags = PG_BUSY;
817:
818: /*
1.46 chs 819: * now do it. note: we must update nextpg in the body of loop or we
1.51 enami 820: * will get stuck. we need to use nextpg if we'll traverse the list
821: * because we may free "pg" before doing the next loop.
1.21 thorpej 822: */
1.22 thorpej 823:
824: if (by_list) {
1.51 enami 825: TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq);
826: nextpg = TAILQ_FIRST(&uobj->memq);
1.53 thorpej 827: PHOLD(curlwp);
1.22 thorpej 828: } else {
829: curoff = start;
1.52 scw 830: nextpg = NULL; /* Quell compiler warning */
1.22 thorpej 831: }
832:
1.46 chs 833: uvm_lock_pageq();
1.22 thorpej 834:
835: /* locked: both page queues and uobj */
1.51 enami 836: for (;;) {
1.22 thorpej 837: if (by_list) {
1.51 enami 838: pg = nextpg;
839: if (pg == &endmp)
840: break;
1.46 chs 841: nextpg = TAILQ_NEXT(pg, listq);
842: if (pg->offset < start || pg->offset >= stop)
1.22 thorpej 843: continue;
844: } else {
1.51 enami 845: if (curoff < stop) {
846: pg = uvm_pagelookup(uobj, curoff);
847: curoff += PAGE_SIZE;
848: } else
849: break;
1.46 chs 850: if (pg == NULL)
1.22 thorpej 851: continue;
852: }
1.46 chs 853: switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
1.41 chs 854:
1.22 thorpej 855: /*
856: * XXX In these first 3 cases, we always just
857: * XXX deactivate the page. We may want to
858: * XXX handle the different cases more specifically
859: * XXX in the future.
860: */
1.46 chs 861:
1.22 thorpej 862: case PGO_CLEANIT|PGO_FREE:
863: case PGO_CLEANIT|PGO_DEACTIVATE:
864: case PGO_DEACTIVATE:
1.25 thorpej 865: deactivate_it:
1.22 thorpej 866: /* skip the page if it's loaned or wired */
1.46 chs 867: if (pg->loan_count != 0 || pg->wire_count != 0)
1.22 thorpej 868: continue;
869:
870: /* ...and deactivate the page. */
1.46 chs 871: pmap_clear_reference(pg);
872: uvm_pagedeactivate(pg);
1.22 thorpej 873: continue;
874:
875: case PGO_FREE:
1.46 chs 876:
1.25 thorpej 877: /*
878: * If there are multiple references to
879: * the object, just deactivate the page.
880: */
1.46 chs 881:
1.25 thorpej 882: if (uobj->uo_refs > 1)
883: goto deactivate_it;
884:
1.22 thorpej 885: /* XXX skip the page if it's loaned or wired */
1.46 chs 886: if (pg->loan_count != 0 || pg->wire_count != 0)
1.22 thorpej 887: continue;
888:
889: /*
1.51 enami 890: * wait and try again if the page is busy.
891: * otherwise free the swap slot and the page.
1.22 thorpej 892: */
1.46 chs 893:
894: pmap_page_protect(pg, VM_PROT_NONE);
1.51 enami 895: if (pg->flags & PG_BUSY) {
896: if (by_list) {
897: TAILQ_INSERT_BEFORE(pg, &curmp, listq);
898: }
1.46 chs 899: pg->flags |= PG_WANTED;
900: uvm_unlock_pageq();
901: UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, 0,
902: "uao_put", 0);
903: simple_lock(&uobj->vmobjlock);
904: uvm_lock_pageq();
1.51 enami 905: if (by_list) {
906: nextpg = TAILQ_NEXT(&curmp, listq);
907: TAILQ_REMOVE(&uobj->memq, &curmp,
908: listq);
909: } else
910: curoff -= PAGE_SIZE;
911: continue;
1.22 thorpej 912: }
1.46 chs 913: uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
914: uvm_pagefree(pg);
1.22 thorpej 915: continue;
916: }
917: }
918: uvm_unlock_pageq();
1.51 enami 919: if (by_list) {
920: TAILQ_REMOVE(&uobj->memq, &endmp, listq);
1.53 thorpej 921: PRELE(curlwp);
1.51 enami 922: }
1.55 ! pk 923: simple_unlock(&uobj->vmobjlock);
1.46 chs 924: return 0;
1.1 mrg 925: }
926:
927: /*
928: * uao_get: fetch me a page
929: *
930: * we have three cases:
931: * 1: page is resident -> just return the page.
932: * 2: page is zero-fill -> allocate a new page and zero it.
933: * 3: page is swapped out -> fetch the page from swap.
934: *
935: * cases 1 and 2 can be handled with PGO_LOCKED, case 3 cannot.
936: * so, if the "center" page hits case 3 (or any page, with PGO_ALLPAGES),
1.40 chs 937: * then we will need to return EBUSY.
1.1 mrg 938: *
939: * => prefer map unlocked (not required)
940: * => object must be locked! we will _unlock_ it before starting any I/O.
941: * => flags: PGO_ALLPAGES: get all of the pages
942: * PGO_LOCKED: fault data structures are locked
943: * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
944: * => NOTE: caller must check for released pages!!
945: */
1.46 chs 946:
1.5 mrg 947: static int
948: uao_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags)
949: struct uvm_object *uobj;
1.28 kleink 950: voff_t offset;
1.5 mrg 951: struct vm_page **pps;
952: int *npagesp;
953: int centeridx, advice, flags;
954: vm_prot_t access_type;
955: {
956: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.28 kleink 957: voff_t current_offset;
1.52 scw 958: struct vm_page *ptmp = NULL; /* Quell compiler warning */
1.46 chs 959: int lcv, gotpages, maxpages, swslot, error, pageidx;
1.5 mrg 960: boolean_t done;
961: UVMHIST_FUNC("uao_get"); UVMHIST_CALLED(pdhist);
962:
1.27 chs 963: UVMHIST_LOG(pdhist, "aobj=%p offset=%d, flags=%d",
964: aobj, offset, flags,0);
1.37 chs 965:
1.5 mrg 966: /*
967: * get number of pages
968: */
1.46 chs 969:
1.5 mrg 970: maxpages = *npagesp;
971:
972: /*
973: * step 1: handled the case where fault data structures are locked.
974: */
1.1 mrg 975:
1.5 mrg 976: if (flags & PGO_LOCKED) {
1.46 chs 977:
1.5 mrg 978: /*
979: * step 1a: get pages that are already resident. only do
980: * this if the data structures are locked (i.e. the first
981: * time through).
982: */
983:
984: done = TRUE; /* be optimistic */
985: gotpages = 0; /* # of pages we got so far */
986: for (lcv = 0, current_offset = offset ; lcv < maxpages ;
987: lcv++, current_offset += PAGE_SIZE) {
988: /* do we care about this page? if not, skip it */
989: if (pps[lcv] == PGO_DONTCARE)
990: continue;
991: ptmp = uvm_pagelookup(uobj, current_offset);
992:
993: /*
1.30 thorpej 994: * if page is new, attempt to allocate the page,
995: * zero-fill'd.
1.5 mrg 996: */
1.46 chs 997:
998: if (ptmp == NULL && uao_find_swslot(&aobj->u_obj,
1.15 chs 999: current_offset >> PAGE_SHIFT) == 0) {
1.5 mrg 1000: ptmp = uvm_pagealloc(uobj, current_offset,
1.30 thorpej 1001: NULL, UVM_PGA_ZERO);
1.5 mrg 1002: if (ptmp) {
1003: /* new page */
1.47 chs 1004: ptmp->flags &= ~(PG_FAKE);
1.5 mrg 1005: ptmp->pqflags |= PQ_AOBJ;
1.47 chs 1006: goto gotpage;
1.5 mrg 1007: }
1008: }
1009:
1010: /*
1.46 chs 1011: * to be useful must get a non-busy page
1.5 mrg 1012: */
1.46 chs 1013:
1014: if (ptmp == NULL || (ptmp->flags & PG_BUSY) != 0) {
1.5 mrg 1015: if (lcv == centeridx ||
1016: (flags & PGO_ALLPAGES) != 0)
1017: /* need to do a wait or I/O! */
1.41 chs 1018: done = FALSE;
1.5 mrg 1019: continue;
1020: }
1021:
1022: /*
1023: * useful page: busy/lock it and plug it in our
1024: * result array
1025: */
1.46 chs 1026:
1.5 mrg 1027: /* caller must un-busy this page */
1.41 chs 1028: ptmp->flags |= PG_BUSY;
1.5 mrg 1029: UVM_PAGE_OWN(ptmp, "uao_get1");
1.47 chs 1030: gotpage:
1.5 mrg 1031: pps[lcv] = ptmp;
1032: gotpages++;
1.46 chs 1033: }
1.5 mrg 1034:
1035: /*
1036: * step 1b: now we've either done everything needed or we
1037: * to unlock and do some waiting or I/O.
1038: */
1039:
1040: UVMHIST_LOG(pdhist, "<- done (done=%d)", done, 0,0,0);
1041: *npagesp = gotpages;
1042: if (done)
1.46 chs 1043: return 0;
1.5 mrg 1044: else
1.46 chs 1045: return EBUSY;
1.1 mrg 1046: }
1047:
1.5 mrg 1048: /*
1049: * step 2: get non-resident or busy pages.
1050: * object is locked. data structures are unlocked.
1051: */
1052:
1053: for (lcv = 0, current_offset = offset ; lcv < maxpages ;
1054: lcv++, current_offset += PAGE_SIZE) {
1.27 chs 1055:
1.5 mrg 1056: /*
1057: * - skip over pages we've already gotten or don't want
1058: * - skip over pages we don't _have_ to get
1059: */
1.27 chs 1060:
1.5 mrg 1061: if (pps[lcv] != NULL ||
1062: (lcv != centeridx && (flags & PGO_ALLPAGES) == 0))
1063: continue;
1064:
1.27 chs 1065: pageidx = current_offset >> PAGE_SHIFT;
1066:
1.5 mrg 1067: /*
1068: * we have yet to locate the current page (pps[lcv]). we
1069: * first look for a page that is already at the current offset.
1070: * if we find a page, we check to see if it is busy or
1071: * released. if that is the case, then we sleep on the page
1072: * until it is no longer busy or released and repeat the lookup.
1073: * if the page we found is neither busy nor released, then we
1074: * busy it (so we own it) and plug it into pps[lcv]. this
1075: * 'break's the following while loop and indicates we are
1076: * ready to move on to the next page in the "lcv" loop above.
1077: *
1078: * if we exit the while loop with pps[lcv] still set to NULL,
1079: * then it means that we allocated a new busy/fake/clean page
1080: * ptmp in the object and we need to do I/O to fill in the data.
1081: */
1082:
1083: /* top of "pps" while loop */
1084: while (pps[lcv] == NULL) {
1085: /* look for a resident page */
1086: ptmp = uvm_pagelookup(uobj, current_offset);
1087:
1088: /* not resident? allocate one now (if we can) */
1089: if (ptmp == NULL) {
1090:
1091: ptmp = uvm_pagealloc(uobj, current_offset,
1.19 chs 1092: NULL, 0);
1.5 mrg 1093:
1094: /* out of RAM? */
1095: if (ptmp == NULL) {
1096: simple_unlock(&uobj->vmobjlock);
1097: UVMHIST_LOG(pdhist,
1098: "sleeping, ptmp == NULL\n",0,0,0,0);
1099: uvm_wait("uao_getpage");
1100: simple_lock(&uobj->vmobjlock);
1.41 chs 1101: continue;
1.5 mrg 1102: }
1103:
1104: /*
1105: * safe with PQ's unlocked: because we just
1106: * alloc'd the page
1107: */
1.46 chs 1108:
1.5 mrg 1109: ptmp->pqflags |= PQ_AOBJ;
1110:
1.41 chs 1111: /*
1.5 mrg 1112: * got new page ready for I/O. break pps while
1113: * loop. pps[lcv] is still NULL.
1114: */
1.46 chs 1115:
1.5 mrg 1116: break;
1117: }
1118:
1119: /* page is there, see if we need to wait on it */
1.46 chs 1120: if ((ptmp->flags & PG_BUSY) != 0) {
1.5 mrg 1121: ptmp->flags |= PG_WANTED;
1122: UVMHIST_LOG(pdhist,
1123: "sleeping, ptmp->flags 0x%x\n",
1124: ptmp->flags,0,0,0);
1.23 thorpej 1125: UVM_UNLOCK_AND_WAIT(ptmp, &uobj->vmobjlock,
1126: FALSE, "uao_get", 0);
1.5 mrg 1127: simple_lock(&uobj->vmobjlock);
1.46 chs 1128: continue;
1.5 mrg 1129: }
1.41 chs 1130:
1131: /*
1.5 mrg 1132: * if we get here then the page has become resident and
1133: * unbusy between steps 1 and 2. we busy it now (so we
1134: * own it) and set pps[lcv] (so that we exit the while
1135: * loop).
1136: */
1.46 chs 1137:
1.5 mrg 1138: /* we own it, caller must un-busy */
1139: ptmp->flags |= PG_BUSY;
1140: UVM_PAGE_OWN(ptmp, "uao_get2");
1141: pps[lcv] = ptmp;
1142: }
1143:
1144: /*
1145: * if we own the valid page at the correct offset, pps[lcv] will
1146: * point to it. nothing more to do except go to the next page.
1147: */
1.46 chs 1148:
1.5 mrg 1149: if (pps[lcv])
1150: continue; /* next lcv */
1151:
1152: /*
1.41 chs 1153: * we have a "fake/busy/clean" page that we just allocated.
1.5 mrg 1154: * do the needed "i/o", either reading from swap or zeroing.
1155: */
1.46 chs 1156:
1157: swslot = uao_find_swslot(&aobj->u_obj, pageidx);
1.5 mrg 1158:
1159: /*
1160: * just zero the page if there's nothing in swap.
1161: */
1.46 chs 1162:
1163: if (swslot == 0) {
1164:
1.5 mrg 1165: /*
1166: * page hasn't existed before, just zero it.
1167: */
1.46 chs 1168:
1.5 mrg 1169: uvm_pagezero(ptmp);
1.27 chs 1170: } else {
1.5 mrg 1171: UVMHIST_LOG(pdhist, "pagein from swslot %d",
1172: swslot, 0,0,0);
1173:
1174: /*
1175: * page in the swapped-out page.
1176: * unlock object for i/o, relock when done.
1177: */
1.46 chs 1178:
1.5 mrg 1179: simple_unlock(&uobj->vmobjlock);
1.46 chs 1180: error = uvm_swap_get(ptmp, swslot, PGO_SYNCIO);
1.5 mrg 1181: simple_lock(&uobj->vmobjlock);
1182:
1183: /*
1184: * I/O done. check for errors.
1185: */
1.46 chs 1186:
1187: if (error != 0) {
1.5 mrg 1188: UVMHIST_LOG(pdhist, "<- done (error=%d)",
1.46 chs 1189: error,0,0,0);
1.5 mrg 1190: if (ptmp->flags & PG_WANTED)
1.24 thorpej 1191: wakeup(ptmp);
1.27 chs 1192:
1193: /*
1194: * remove the swap slot from the aobj
1195: * and mark the aobj as having no real slot.
1196: * don't free the swap slot, thus preventing
1197: * it from being used again.
1198: */
1.46 chs 1199:
1.27 chs 1200: swslot = uao_set_swslot(&aobj->u_obj, pageidx,
1201: SWSLOT_BAD);
1.45 chs 1202: if (swslot != -1) {
1203: uvm_swap_markbad(swslot, 1);
1204: }
1.27 chs 1205:
1.5 mrg 1206: uvm_lock_pageq();
1207: uvm_pagefree(ptmp);
1208: uvm_unlock_pageq();
1209: simple_unlock(&uobj->vmobjlock);
1.46 chs 1210: return error;
1.5 mrg 1211: }
1212: }
1213:
1.41 chs 1214: /*
1.5 mrg 1215: * we got the page! clear the fake flag (indicates valid
1216: * data now in page) and plug into our result array. note
1.41 chs 1217: * that page is still busy.
1.5 mrg 1218: *
1219: * it is the callers job to:
1220: * => check if the page is released
1221: * => unbusy the page
1222: * => activate the page
1223: */
1224:
1.46 chs 1225: ptmp->flags &= ~PG_FAKE;
1.5 mrg 1226: pps[lcv] = ptmp;
1.46 chs 1227: }
1.1 mrg 1228:
1229: /*
1.5 mrg 1230: * finally, unlock object and return.
1231: */
1.1 mrg 1232:
1233: simple_unlock(&uobj->vmobjlock);
1.5 mrg 1234: UVMHIST_LOG(pdhist, "<- done (OK)",0,0,0,0);
1.46 chs 1235: return 0;
1.1 mrg 1236: }
1237:
1238: /*
1.18 chs 1239: * uao_dropswap: release any swap resources from this aobj page.
1.41 chs 1240: *
1.18 chs 1241: * => aobj must be locked or have a reference count of 0.
1242: */
1243:
1244: void
1245: uao_dropswap(uobj, pageidx)
1246: struct uvm_object *uobj;
1247: int pageidx;
1248: {
1249: int slot;
1250:
1251: slot = uao_set_swslot(uobj, pageidx, 0);
1252: if (slot) {
1253: uvm_swap_free(slot, 1);
1254: }
1.27 chs 1255: }
1256:
1257: /*
1258: * page in every page in every aobj that is paged-out to a range of swslots.
1.41 chs 1259: *
1.27 chs 1260: * => nothing should be locked.
1261: * => returns TRUE if pagein was aborted due to lack of memory.
1262: */
1.46 chs 1263:
1.27 chs 1264: boolean_t
1265: uao_swap_off(startslot, endslot)
1266: int startslot, endslot;
1267: {
1268: struct uvm_aobj *aobj, *nextaobj;
1.46 chs 1269: boolean_t rv;
1.27 chs 1270:
1271: /*
1272: * walk the list of all aobjs.
1273: */
1274:
1275: restart:
1276: simple_lock(&uao_list_lock);
1277: for (aobj = LIST_FIRST(&uao_list);
1278: aobj != NULL;
1279: aobj = nextaobj) {
1280:
1281: /*
1.46 chs 1282: * try to get the object lock, start all over if we fail.
1.27 chs 1283: * most of the time we'll get the aobj lock,
1284: * so this should be a rare case.
1285: */
1.46 chs 1286:
1.27 chs 1287: if (!simple_lock_try(&aobj->u_obj.vmobjlock)) {
1288: simple_unlock(&uao_list_lock);
1289: goto restart;
1290: }
1291:
1292: /*
1293: * add a ref to the aobj so it doesn't disappear
1294: * while we're working.
1295: */
1.46 chs 1296:
1.27 chs 1297: uao_reference_locked(&aobj->u_obj);
1298:
1299: /*
1300: * now it's safe to unlock the uao list.
1301: */
1.46 chs 1302:
1.27 chs 1303: simple_unlock(&uao_list_lock);
1304:
1305: /*
1306: * page in any pages in the swslot range.
1307: * if there's an error, abort and return the error.
1308: */
1.46 chs 1309:
1.27 chs 1310: rv = uao_pagein(aobj, startslot, endslot);
1311: if (rv) {
1312: uao_detach_locked(&aobj->u_obj);
1313: return rv;
1314: }
1315:
1316: /*
1317: * we're done with this aobj.
1318: * relock the list and drop our ref on the aobj.
1319: */
1.46 chs 1320:
1.27 chs 1321: simple_lock(&uao_list_lock);
1322: nextaobj = LIST_NEXT(aobj, u_list);
1323: uao_detach_locked(&aobj->u_obj);
1324: }
1325:
1326: /*
1327: * done with traversal, unlock the list
1328: */
1329: simple_unlock(&uao_list_lock);
1330: return FALSE;
1331: }
1332:
1333:
1334: /*
1335: * page in any pages from aobj in the given range.
1336: *
1337: * => aobj must be locked and is returned locked.
1338: * => returns TRUE if pagein was aborted due to lack of memory.
1339: */
1340: static boolean_t
1341: uao_pagein(aobj, startslot, endslot)
1342: struct uvm_aobj *aobj;
1343: int startslot, endslot;
1344: {
1345: boolean_t rv;
1346:
1347: if (UAO_USES_SWHASH(aobj)) {
1348: struct uao_swhash_elt *elt;
1349: int bucket;
1350:
1351: restart:
1352: for (bucket = aobj->u_swhashmask; bucket >= 0; bucket--) {
1353: for (elt = LIST_FIRST(&aobj->u_swhash[bucket]);
1354: elt != NULL;
1355: elt = LIST_NEXT(elt, list)) {
1356: int i;
1357:
1358: for (i = 0; i < UAO_SWHASH_CLUSTER_SIZE; i++) {
1359: int slot = elt->slots[i];
1360:
1361: /*
1362: * if the slot isn't in range, skip it.
1363: */
1.46 chs 1364:
1.41 chs 1365: if (slot < startslot ||
1.27 chs 1366: slot >= endslot) {
1367: continue;
1368: }
1369:
1370: /*
1371: * process the page,
1372: * the start over on this object
1373: * since the swhash elt
1374: * may have been freed.
1375: */
1.46 chs 1376:
1.27 chs 1377: rv = uao_pagein_page(aobj,
1378: UAO_SWHASH_ELT_PAGEIDX_BASE(elt) + i);
1379: if (rv) {
1380: return rv;
1381: }
1382: goto restart;
1383: }
1384: }
1385: }
1386: } else {
1387: int i;
1388:
1389: for (i = 0; i < aobj->u_pages; i++) {
1390: int slot = aobj->u_swslots[i];
1391:
1392: /*
1393: * if the slot isn't in range, skip it
1394: */
1.46 chs 1395:
1.27 chs 1396: if (slot < startslot || slot >= endslot) {
1397: continue;
1398: }
1399:
1400: /*
1401: * process the page.
1402: */
1.46 chs 1403:
1.27 chs 1404: rv = uao_pagein_page(aobj, i);
1405: if (rv) {
1406: return rv;
1407: }
1408: }
1409: }
1410:
1411: return FALSE;
1412: }
1413:
1414: /*
1415: * page in a page from an aobj. used for swap_off.
1416: * returns TRUE if pagein was aborted due to lack of memory.
1417: *
1418: * => aobj must be locked and is returned locked.
1419: */
1.46 chs 1420:
1.27 chs 1421: static boolean_t
1422: uao_pagein_page(aobj, pageidx)
1423: struct uvm_aobj *aobj;
1424: int pageidx;
1425: {
1426: struct vm_page *pg;
1427: int rv, slot, npages;
1428:
1429: pg = NULL;
1430: npages = 1;
1431: /* locked: aobj */
1432: rv = uao_get(&aobj->u_obj, pageidx << PAGE_SHIFT,
1433: &pg, &npages, 0, VM_PROT_READ|VM_PROT_WRITE, 0, 0);
1434: /* unlocked: aobj */
1435:
1436: /*
1437: * relock and finish up.
1438: */
1.46 chs 1439:
1.27 chs 1440: simple_lock(&aobj->u_obj.vmobjlock);
1441: switch (rv) {
1.40 chs 1442: case 0:
1.27 chs 1443: break;
1444:
1.40 chs 1445: case EIO:
1446: case ERESTART:
1.46 chs 1447:
1.27 chs 1448: /*
1449: * nothing more to do on errors.
1.40 chs 1450: * ERESTART can only mean that the anon was freed,
1.27 chs 1451: * so again there's nothing to do.
1452: */
1.46 chs 1453:
1.27 chs 1454: return FALSE;
1455: }
1456:
1457: /*
1458: * ok, we've got the page now.
1459: * mark it as dirty, clear its swslot and un-busy it.
1460: */
1.46 chs 1461:
1.27 chs 1462: slot = uao_set_swslot(&aobj->u_obj, pageidx, 0);
1463: uvm_swap_free(slot, 1);
1464: pg->flags &= ~(PG_BUSY|PG_CLEAN|PG_FAKE);
1465: UVM_PAGE_OWN(pg, NULL);
1466:
1467: /*
1.46 chs 1468: * deactivate the page (to make sure it's on a page queue).
1.27 chs 1469: */
1.46 chs 1470:
1.27 chs 1471: uvm_lock_pageq();
1472: uvm_pagedeactivate(pg);
1473: uvm_unlock_pageq();
1474: return FALSE;
1.1 mrg 1475: }
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to uvm_aobj.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / uvm