Annotation of src/sys/uvm/uvm_aobj.c, Revision 1.31
1.31 ! thorpej 1: /* $NetBSD: uvm_aobj.c,v 1.30 2000/04/10 00:32:46 thorpej Exp $ */
1.6 mrg 2:
1.7 chs 3: /*
4: * Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
5: * Washington University.
6: * All rights reserved.
7: *
8: * Redistribution and use in source and binary forms, with or without
9: * modification, are permitted provided that the following conditions
10: * are met:
11: * 1. Redistributions of source code must retain the above copyright
12: * notice, this list of conditions and the following disclaimer.
13: * 2. Redistributions in binary form must reproduce the above copyright
14: * notice, this list of conditions and the following disclaimer in the
15: * documentation and/or other materials provided with the distribution.
16: * 3. All advertising materials mentioning features or use of this software
17: * must display the following acknowledgement:
18: * This product includes software developed by Charles D. Cranor and
19: * Washington University.
20: * 4. The name of the author may not be used to endorse or promote products
21: * derived from this software without specific prior written permission.
22: *
23: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33: *
1.4 mrg 34: * from: Id: uvm_aobj.c,v 1.1.2.5 1998/02/06 05:14:38 chs Exp
35: */
1.7 chs 36: /*
37: * uvm_aobj.c: anonymous memory uvm_object pager
38: *
39: * author: Chuck Silvers <chuq@chuq.com>
40: * started: Jan-1998
41: *
42: * - design mostly from Chuck Cranor
43: */
44:
45:
46:
47: #include "opt_uvmhist.h"
1.1 mrg 48:
49: #include <sys/param.h>
50: #include <sys/systm.h>
51: #include <sys/proc.h>
52: #include <sys/malloc.h>
1.12 thorpej 53: #include <sys/pool.h>
1.27 chs 54: #include <sys/kernel.h>
1.1 mrg 55:
56: #include <vm/vm.h>
57: #include <vm/vm_page.h>
58: #include <vm/vm_kern.h>
59:
60: #include <uvm/uvm.h>
61:
62: /*
63: * an aobj manages anonymous-memory backed uvm_objects. in addition
64: * to keeping the list of resident pages, it also keeps a list of
65: * allocated swap blocks. depending on the size of the aobj this list
66: * of allocated swap blocks is either stored in an array (small objects)
67: * or in a hash table (large objects).
68: */
69:
70: /*
71: * local structures
72: */
73:
74: /*
75: * for hash tables, we break the address space of the aobj into blocks
76: * of UAO_SWHASH_CLUSTER_SIZE pages. we require the cluster size to
77: * be a power of two.
78: */
79:
80: #define UAO_SWHASH_CLUSTER_SHIFT 4
81: #define UAO_SWHASH_CLUSTER_SIZE (1 << UAO_SWHASH_CLUSTER_SHIFT)
82:
83: /* get the "tag" for this page index */
84: #define UAO_SWHASH_ELT_TAG(PAGEIDX) \
85: ((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT)
86:
87: /* given an ELT and a page index, find the swap slot */
88: #define UAO_SWHASH_ELT_PAGESLOT(ELT, PAGEIDX) \
89: ((ELT)->slots[(PAGEIDX) & (UAO_SWHASH_CLUSTER_SIZE - 1)])
90:
91: /* given an ELT, return its pageidx base */
92: #define UAO_SWHASH_ELT_PAGEIDX_BASE(ELT) \
93: ((ELT)->tag << UAO_SWHASH_CLUSTER_SHIFT)
94:
95: /*
96: * the swhash hash function
97: */
98: #define UAO_SWHASH_HASH(AOBJ, PAGEIDX) \
99: (&(AOBJ)->u_swhash[(((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT) \
100: & (AOBJ)->u_swhashmask)])
101:
102: /*
103: * the swhash threshhold determines if we will use an array or a
104: * hash table to store the list of allocated swap blocks.
105: */
106:
107: #define UAO_SWHASH_THRESHOLD (UAO_SWHASH_CLUSTER_SIZE * 4)
108: #define UAO_USES_SWHASH(AOBJ) \
109: ((AOBJ)->u_pages > UAO_SWHASH_THRESHOLD) /* use hash? */
110:
111: /*
1.3 chs 112: * the number of buckets in a swhash, with an upper bound
1.1 mrg 113: */
114: #define UAO_SWHASH_MAXBUCKETS 256
115: #define UAO_SWHASH_BUCKETS(AOBJ) \
116: (min((AOBJ)->u_pages >> UAO_SWHASH_CLUSTER_SHIFT, \
117: UAO_SWHASH_MAXBUCKETS))
118:
119:
120: /*
121: * uao_swhash_elt: when a hash table is being used, this structure defines
122: * the format of an entry in the bucket list.
123: */
124:
125: struct uao_swhash_elt {
1.5 mrg 126: LIST_ENTRY(uao_swhash_elt) list; /* the hash list */
1.28 kleink 127: voff_t tag; /* our 'tag' */
1.5 mrg 128: int count; /* our number of active slots */
129: int slots[UAO_SWHASH_CLUSTER_SIZE]; /* the slots */
1.1 mrg 130: };
131:
132: /*
133: * uao_swhash: the swap hash table structure
134: */
135:
136: LIST_HEAD(uao_swhash, uao_swhash_elt);
137:
1.12 thorpej 138: /*
139: * uao_swhash_elt_pool: pool of uao_swhash_elt structures
140: */
141:
142: struct pool uao_swhash_elt_pool;
1.1 mrg 143:
144: /*
145: * uvm_aobj: the actual anon-backed uvm_object
146: *
147: * => the uvm_object is at the top of the structure, this allows
148: * (struct uvm_device *) == (struct uvm_object *)
149: * => only one of u_swslots and u_swhash is used in any given aobj
150: */
151:
152: struct uvm_aobj {
1.5 mrg 153: struct uvm_object u_obj; /* has: lock, pgops, memq, #pages, #refs */
1.11 drochner 154: int u_pages; /* number of pages in entire object */
1.5 mrg 155: int u_flags; /* the flags (see uvm_aobj.h) */
156: int *u_swslots; /* array of offset->swapslot mappings */
157: /*
158: * hashtable of offset->swapslot mappings
159: * (u_swhash is an array of bucket heads)
160: */
161: struct uao_swhash *u_swhash;
162: u_long u_swhashmask; /* mask for hashtable */
163: LIST_ENTRY(uvm_aobj) u_list; /* global list of aobjs */
1.1 mrg 164: };
165:
166: /*
1.12 thorpej 167: * uvm_aobj_pool: pool of uvm_aobj structures
168: */
169:
170: struct pool uvm_aobj_pool;
171:
172: /*
1.1 mrg 173: * local functions
174: */
175:
176: static struct uao_swhash_elt *uao_find_swhash_elt __P((struct uvm_aobj *,
177: int, boolean_t));
1.28 kleink 178: static int uao_find_swslot __P((struct uvm_aobj *, int));
179: static boolean_t uao_flush __P((struct uvm_object *,
180: voff_t, voff_t, int));
1.1 mrg 181: static void uao_free __P((struct uvm_aobj *));
1.28 kleink 182: static int uao_get __P((struct uvm_object *, voff_t,
183: vm_page_t *, int *, int,
1.1 mrg 184: vm_prot_t, int, int));
1.28 kleink 185: static boolean_t uao_releasepg __P((struct vm_page *,
1.1 mrg 186: struct vm_page **));
1.27 chs 187: static boolean_t uao_pagein __P((struct uvm_aobj *, int, int));
188: static boolean_t uao_pagein_page __P((struct uvm_aobj *, int));
1.1 mrg 189:
190:
191:
192: /*
193: * aobj_pager
194: *
195: * note that some functions (e.g. put) are handled elsewhere
196: */
197:
198: struct uvm_pagerops aobj_pager = {
1.27 chs 199: NULL, /* init */
1.5 mrg 200: uao_reference, /* reference */
201: uao_detach, /* detach */
202: NULL, /* fault */
203: uao_flush, /* flush */
204: uao_get, /* get */
205: NULL, /* asyncget */
206: NULL, /* put (done by pagedaemon) */
207: NULL, /* cluster */
208: NULL, /* mk_pcluster */
209: NULL, /* aiodone */
210: uao_releasepg /* releasepg */
1.1 mrg 211: };
212:
213: /*
214: * uao_list: global list of active aobjs, locked by uao_list_lock
215: */
216:
217: static LIST_HEAD(aobjlist, uvm_aobj) uao_list;
218: static simple_lock_data_t uao_list_lock;
219:
220:
221: /*
222: * functions
223: */
224:
225: /*
226: * hash table/array related functions
227: */
228:
229: /*
230: * uao_find_swhash_elt: find (or create) a hash table entry for a page
231: * offset.
232: *
233: * => the object should be locked by the caller
234: */
235:
1.5 mrg 236: static struct uao_swhash_elt *
237: uao_find_swhash_elt(aobj, pageidx, create)
238: struct uvm_aobj *aobj;
239: int pageidx;
240: boolean_t create;
241: {
242: struct uao_swhash *swhash;
243: struct uao_swhash_elt *elt;
1.28 kleink 244: voff_t page_tag;
1.1 mrg 245:
1.5 mrg 246: swhash = UAO_SWHASH_HASH(aobj, pageidx); /* first hash to get bucket */
247: page_tag = UAO_SWHASH_ELT_TAG(pageidx); /* tag to search for */
1.1 mrg 248:
1.5 mrg 249: /*
250: * now search the bucket for the requested tag
251: */
252: for (elt = swhash->lh_first; elt != NULL; elt = elt->list.le_next) {
253: if (elt->tag == page_tag)
254: return(elt);
255: }
256:
257: /* fail now if we are not allowed to create a new entry in the bucket */
258: if (!create)
259: return NULL;
260:
261:
262: /*
1.12 thorpej 263: * allocate a new entry for the bucket and init/insert it in
1.5 mrg 264: */
1.12 thorpej 265: elt = pool_get(&uao_swhash_elt_pool, PR_WAITOK);
1.5 mrg 266: LIST_INSERT_HEAD(swhash, elt, list);
267: elt->tag = page_tag;
268: elt->count = 0;
1.9 perry 269: memset(elt->slots, 0, sizeof(elt->slots));
1.5 mrg 270:
271: return(elt);
1.1 mrg 272: }
273:
274: /*
275: * uao_find_swslot: find the swap slot number for an aobj/pageidx
276: *
277: * => object must be locked by caller
278: */
1.5 mrg 279: __inline static int
280: uao_find_swslot(aobj, pageidx)
281: struct uvm_aobj *aobj;
1.11 drochner 282: int pageidx;
1.1 mrg 283: {
284:
1.5 mrg 285: /*
286: * if noswap flag is set, then we never return a slot
287: */
1.1 mrg 288:
1.5 mrg 289: if (aobj->u_flags & UAO_FLAG_NOSWAP)
290: return(0);
1.1 mrg 291:
1.5 mrg 292: /*
293: * if hashing, look in hash table.
294: */
1.1 mrg 295:
1.5 mrg 296: if (UAO_USES_SWHASH(aobj)) {
297: struct uao_swhash_elt *elt =
298: uao_find_swhash_elt(aobj, pageidx, FALSE);
299:
300: if (elt)
301: return(UAO_SWHASH_ELT_PAGESLOT(elt, pageidx));
302: else
1.31 ! thorpej 303: return(0);
1.5 mrg 304: }
1.1 mrg 305:
1.5 mrg 306: /*
307: * otherwise, look in the array
308: */
309: return(aobj->u_swslots[pageidx]);
1.1 mrg 310: }
311:
312: /*
313: * uao_set_swslot: set the swap slot for a page in an aobj.
314: *
315: * => setting a slot to zero frees the slot
316: * => object must be locked by caller
317: */
1.5 mrg 318: int
319: uao_set_swslot(uobj, pageidx, slot)
320: struct uvm_object *uobj;
321: int pageidx, slot;
322: {
323: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
324: int oldslot;
325: UVMHIST_FUNC("uao_set_swslot"); UVMHIST_CALLED(pdhist);
326: UVMHIST_LOG(pdhist, "aobj %p pageidx %d slot %d",
327: aobj, pageidx, slot, 0);
1.1 mrg 328:
1.5 mrg 329: /*
330: * if noswap flag is set, then we can't set a slot
331: */
1.1 mrg 332:
1.5 mrg 333: if (aobj->u_flags & UAO_FLAG_NOSWAP) {
1.1 mrg 334:
1.5 mrg 335: if (slot == 0)
336: return(0); /* a clear is ok */
1.1 mrg 337:
1.5 mrg 338: /* but a set is not */
339: printf("uao_set_swslot: uobj = %p\n", uobj);
340: panic("uao_set_swslot: attempt to set a slot on a NOSWAP object");
341: }
1.1 mrg 342:
1.5 mrg 343: /*
344: * are we using a hash table? if so, add it in the hash.
345: */
1.1 mrg 346:
1.5 mrg 347: if (UAO_USES_SWHASH(aobj)) {
1.12 thorpej 348: /*
349: * Avoid allocating an entry just to free it again if
350: * the page had not swap slot in the first place, and
351: * we are freeing.
352: */
1.5 mrg 353: struct uao_swhash_elt *elt =
1.12 thorpej 354: uao_find_swhash_elt(aobj, pageidx, slot ? TRUE : FALSE);
355: if (elt == NULL) {
356: #ifdef DIAGNOSTIC
357: if (slot)
358: panic("uao_set_swslot: didn't create elt");
359: #endif
360: return (0);
361: }
1.5 mrg 362:
363: oldslot = UAO_SWHASH_ELT_PAGESLOT(elt, pageidx);
364: UAO_SWHASH_ELT_PAGESLOT(elt, pageidx) = slot;
365:
366: /*
367: * now adjust the elt's reference counter and free it if we've
368: * dropped it to zero.
369: */
370:
371: /* an allocation? */
372: if (slot) {
373: if (oldslot == 0)
374: elt->count++;
375: } else { /* freeing slot ... */
376: if (oldslot) /* to be safe */
377: elt->count--;
378:
379: if (elt->count == 0) {
380: LIST_REMOVE(elt, list);
1.12 thorpej 381: pool_put(&uao_swhash_elt_pool, elt);
1.5 mrg 382: }
383: }
384:
385: } else {
386: /* we are using an array */
387: oldslot = aobj->u_swslots[pageidx];
388: aobj->u_swslots[pageidx] = slot;
389: }
390: return (oldslot);
1.1 mrg 391: }
392:
393: /*
394: * end of hash/array functions
395: */
396:
397: /*
398: * uao_free: free all resources held by an aobj, and then free the aobj
399: *
400: * => the aobj should be dead
401: */
402: static void
403: uao_free(aobj)
1.5 mrg 404: struct uvm_aobj *aobj;
1.1 mrg 405: {
406:
1.27 chs 407: simple_unlock(&aobj->u_obj.vmobjlock);
408:
1.5 mrg 409: if (UAO_USES_SWHASH(aobj)) {
410: int i, hashbuckets = aobj->u_swhashmask + 1;
1.1 mrg 411:
1.5 mrg 412: /*
413: * free the swslots from each hash bucket,
414: * then the hash bucket, and finally the hash table itself.
415: */
416: for (i = 0; i < hashbuckets; i++) {
417: struct uao_swhash_elt *elt, *next;
418:
1.27 chs 419: for (elt = LIST_FIRST(&aobj->u_swhash[i]);
420: elt != NULL;
421: elt = next) {
1.5 mrg 422: int j;
423:
1.27 chs 424: for (j = 0; j < UAO_SWHASH_CLUSTER_SIZE; j++) {
1.5 mrg 425: int slot = elt->slots[j];
426:
1.18 chs 427: if (slot) {
1.5 mrg 428: uvm_swap_free(slot, 1);
1.18 chs 429:
430: /*
431: * this page is no longer
432: * only in swap.
433: */
434: simple_lock(&uvm.swap_data_lock);
435: uvmexp.swpgonly--;
436: simple_unlock(&uvm.swap_data_lock);
437: }
1.5 mrg 438: }
439:
1.27 chs 440: next = LIST_NEXT(elt, list);
1.12 thorpej 441: pool_put(&uao_swhash_elt_pool, elt);
1.5 mrg 442: }
443: }
444: FREE(aobj->u_swhash, M_UVMAOBJ);
445: } else {
446: int i;
447:
448: /*
449: * free the array
450: */
451:
1.27 chs 452: for (i = 0; i < aobj->u_pages; i++) {
1.5 mrg 453: int slot = aobj->u_swslots[i];
454:
1.18 chs 455: if (slot) {
1.5 mrg 456: uvm_swap_free(slot, 1);
1.18 chs 457:
458: /* this page is no longer only in swap. */
459: simple_lock(&uvm.swap_data_lock);
460: uvmexp.swpgonly--;
461: simple_unlock(&uvm.swap_data_lock);
462: }
1.5 mrg 463: }
464: FREE(aobj->u_swslots, M_UVMAOBJ);
1.1 mrg 465: }
466:
1.5 mrg 467: /*
468: * finally free the aobj itself
469: */
1.12 thorpej 470: pool_put(&uvm_aobj_pool, aobj);
1.1 mrg 471: }
472:
473: /*
474: * pager functions
475: */
476:
477: /*
478: * uao_create: create an aobj of the given size and return its uvm_object.
479: *
480: * => for normal use, flags are always zero
481: * => for the kernel object, the flags are:
482: * UAO_FLAG_KERNOBJ - allocate the kernel object (can only happen once)
483: * UAO_FLAG_KERNSWAP - enable swapping of kernel object (" ")
484: */
1.5 mrg 485: struct uvm_object *
486: uao_create(size, flags)
1.10 eeh 487: vsize_t size;
1.5 mrg 488: int flags;
489: {
1.27 chs 490: static struct uvm_aobj kernel_object_store; /* home of kernel_object */
1.5 mrg 491: static int kobj_alloced = 0; /* not allocated yet */
1.15 chs 492: int pages = round_page(size) >> PAGE_SHIFT;
1.5 mrg 493: struct uvm_aobj *aobj;
1.1 mrg 494:
1.5 mrg 495: /*
1.27 chs 496: * malloc a new aobj unless we are asked for the kernel object
497: */
1.5 mrg 498: if (flags & UAO_FLAG_KERNOBJ) { /* want kernel object? */
499: if (kobj_alloced)
500: panic("uao_create: kernel object already allocated");
501:
502: aobj = &kernel_object_store;
503: aobj->u_pages = pages;
504: aobj->u_flags = UAO_FLAG_NOSWAP; /* no swap to start */
505: /* we are special, we never die */
506: aobj->u_obj.uo_refs = UVM_OBJ_KERN;
507: kobj_alloced = UAO_FLAG_KERNOBJ;
508: } else if (flags & UAO_FLAG_KERNSWAP) {
509: aobj = &kernel_object_store;
510: if (kobj_alloced != UAO_FLAG_KERNOBJ)
511: panic("uao_create: asked to enable swap on kernel object");
512: kobj_alloced = UAO_FLAG_KERNSWAP;
513: } else { /* normal object */
1.12 thorpej 514: aobj = pool_get(&uvm_aobj_pool, PR_WAITOK);
1.5 mrg 515: aobj->u_pages = pages;
516: aobj->u_flags = 0; /* normal object */
517: aobj->u_obj.uo_refs = 1; /* start with 1 reference */
518: }
1.1 mrg 519:
1.5 mrg 520: /*
521: * allocate hash/array if necessary
522: *
523: * note: in the KERNSWAP case no need to worry about locking since
524: * we are still booting we should be the only thread around.
525: */
526: if (flags == 0 || (flags & UAO_FLAG_KERNSWAP) != 0) {
527: int mflags = (flags & UAO_FLAG_KERNSWAP) != 0 ?
528: M_NOWAIT : M_WAITOK;
529:
530: /* allocate hash table or array depending on object size */
1.27 chs 531: if (UAO_USES_SWHASH(aobj)) {
1.5 mrg 532: aobj->u_swhash = hashinit(UAO_SWHASH_BUCKETS(aobj),
533: M_UVMAOBJ, mflags, &aobj->u_swhashmask);
534: if (aobj->u_swhash == NULL)
535: panic("uao_create: hashinit swhash failed");
536: } else {
537: MALLOC(aobj->u_swslots, int *, pages * sizeof(int),
538: M_UVMAOBJ, mflags);
539: if (aobj->u_swslots == NULL)
540: panic("uao_create: malloc swslots failed");
1.9 perry 541: memset(aobj->u_swslots, 0, pages * sizeof(int));
1.5 mrg 542: }
543:
544: if (flags) {
545: aobj->u_flags &= ~UAO_FLAG_NOSWAP; /* clear noswap */
546: return(&aobj->u_obj);
547: /* done! */
548: }
549: }
550:
551: /*
552: * init aobj fields
553: */
554: simple_lock_init(&aobj->u_obj.vmobjlock);
555: aobj->u_obj.pgops = &aobj_pager;
556: TAILQ_INIT(&aobj->u_obj.memq);
557: aobj->u_obj.uo_npages = 0;
1.1 mrg 558:
1.5 mrg 559: /*
560: * now that aobj is ready, add it to the global list
561: */
562: simple_lock(&uao_list_lock);
563: LIST_INSERT_HEAD(&uao_list, aobj, u_list);
564: simple_unlock(&uao_list_lock);
565:
566: /*
567: * done!
568: */
569: return(&aobj->u_obj);
1.1 mrg 570: }
571:
572:
573:
574: /*
575: * uao_init: set up aobj pager subsystem
576: *
577: * => called at boot time from uvm_pager_init()
578: */
1.27 chs 579: void
1.5 mrg 580: uao_init()
581: {
1.12 thorpej 582: static int uao_initialized;
583:
584: if (uao_initialized)
585: return;
586: uao_initialized = TRUE;
1.1 mrg 587:
1.5 mrg 588: LIST_INIT(&uao_list);
589: simple_lock_init(&uao_list_lock);
1.12 thorpej 590:
1.14 thorpej 591: /*
592: * NOTE: Pages fror this pool must not come from a pageable
593: * kernel map!
594: */
1.12 thorpej 595: pool_init(&uao_swhash_elt_pool, sizeof(struct uao_swhash_elt),
1.13 thorpej 596: 0, 0, 0, "uaoeltpl", 0, NULL, NULL, M_UVMAOBJ);
1.12 thorpej 597:
598: pool_init(&uvm_aobj_pool, sizeof(struct uvm_aobj), 0, 0, 0,
599: "aobjpl", 0,
600: pool_page_alloc_nointr, pool_page_free_nointr, M_UVMAOBJ);
1.1 mrg 601: }
602:
603: /*
604: * uao_reference: add a ref to an aobj
605: *
1.27 chs 606: * => aobj must be unlocked
607: * => just lock it and call the locked version
1.1 mrg 608: */
1.5 mrg 609: void
610: uao_reference(uobj)
611: struct uvm_object *uobj;
1.1 mrg 612: {
1.27 chs 613: simple_lock(&uobj->vmobjlock);
614: uao_reference_locked(uobj);
615: simple_unlock(&uobj->vmobjlock);
616: }
617:
618: /*
619: * uao_reference_locked: add a ref to an aobj that is already locked
620: *
621: * => aobj must be locked
622: * this needs to be separate from the normal routine
623: * since sometimes we need to add a reference to an aobj when
624: * it's already locked.
625: */
626: void
627: uao_reference_locked(uobj)
628: struct uvm_object *uobj;
629: {
1.5 mrg 630: UVMHIST_FUNC("uao_reference"); UVMHIST_CALLED(maphist);
1.1 mrg 631:
1.5 mrg 632: /*
633: * kernel_object already has plenty of references, leave it alone.
634: */
1.1 mrg 635:
1.20 thorpej 636: if (UVM_OBJ_IS_KERN_OBJECT(uobj))
1.5 mrg 637: return;
1.1 mrg 638:
1.5 mrg 639: uobj->uo_refs++; /* bump! */
640: UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)",
1.27 chs 641: uobj, uobj->uo_refs,0,0);
1.1 mrg 642: }
643:
1.27 chs 644:
1.1 mrg 645: /*
646: * uao_detach: drop a reference to an aobj
647: *
1.27 chs 648: * => aobj must be unlocked
649: * => just lock it and call the locked version
1.1 mrg 650: */
1.5 mrg 651: void
652: uao_detach(uobj)
653: struct uvm_object *uobj;
654: {
1.27 chs 655: simple_lock(&uobj->vmobjlock);
656: uao_detach_locked(uobj);
657: }
658:
659:
660: /*
661: * uao_detach_locked: drop a reference to an aobj
662: *
663: * => aobj must be locked, and is unlocked (or freed) upon return.
664: * this needs to be separate from the normal routine
665: * since sometimes we need to detach from an aobj when
666: * it's already locked.
667: */
668: void
669: uao_detach_locked(uobj)
670: struct uvm_object *uobj;
671: {
1.5 mrg 672: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
673: struct vm_page *pg;
674: boolean_t busybody;
675: UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist);
1.1 mrg 676:
1.5 mrg 677: /*
678: * detaching from kernel_object is a noop.
679: */
1.27 chs 680: if (UVM_OBJ_IS_KERN_OBJECT(uobj)) {
681: simple_unlock(&uobj->vmobjlock);
1.5 mrg 682: return;
1.27 chs 683: }
1.5 mrg 684:
685: UVMHIST_LOG(maphist," (uobj=0x%x) ref=%d", uobj,uobj->uo_refs,0,0);
686: uobj->uo_refs--; /* drop ref! */
687: if (uobj->uo_refs) { /* still more refs? */
688: simple_unlock(&uobj->vmobjlock);
689: UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
690: return;
691: }
692:
693: /*
694: * remove the aobj from the global list.
695: */
696: simple_lock(&uao_list_lock);
697: LIST_REMOVE(aobj, u_list);
698: simple_unlock(&uao_list_lock);
699:
700: /*
1.27 chs 701: * free all the pages that aren't PG_BUSY,
702: * mark for release any that are.
1.5 mrg 703: */
704: busybody = FALSE;
1.27 chs 705: for (pg = TAILQ_FIRST(&uobj->memq);
706: pg != NULL;
707: pg = TAILQ_NEXT(pg, listq)) {
1.5 mrg 708: if (pg->flags & PG_BUSY) {
709: pg->flags |= PG_RELEASED;
710: busybody = TRUE;
711: continue;
712: }
713:
714: /* zap the mappings, free the swap slot, free the page */
1.26 chs 715: pmap_page_protect(pg, VM_PROT_NONE);
1.18 chs 716: uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
1.5 mrg 717: uvm_lock_pageq();
718: uvm_pagefree(pg);
719: uvm_unlock_pageq();
720: }
1.1 mrg 721:
1.5 mrg 722: /*
723: * if we found any busy pages, we're done for now.
724: * mark the aobj for death, releasepg will finish up for us.
725: */
726: if (busybody) {
727: aobj->u_flags |= UAO_FLAG_KILLME;
728: simple_unlock(&aobj->u_obj.vmobjlock);
729: return;
730: }
1.1 mrg 731:
1.5 mrg 732: /*
733: * finally, free the rest.
734: */
735: uao_free(aobj);
736: }
1.1 mrg 737:
738: /*
1.22 thorpej 739: * uao_flush: "flush" pages out of a uvm object
740: *
741: * => object should be locked by caller. we may _unlock_ the object
742: * if (and only if) we need to clean a page (PGO_CLEANIT).
743: * XXXJRT Currently, however, we don't. In the case of cleaning
744: * XXXJRT a page, we simply just deactivate it. Should probably
745: * XXXJRT handle this better, in the future (although "flushing"
746: * XXXJRT anonymous memory isn't terribly important).
747: * => if PGO_CLEANIT is not set, then we will neither unlock the object
748: * or block.
749: * => if PGO_ALLPAGE is set, then all pages in the object are valid targets
750: * for flushing.
751: * => NOTE: we rely on the fact that the object's memq is a TAILQ and
752: * that new pages are inserted on the tail end of the list. thus,
753: * we can make a complete pass through the object in one go by starting
754: * at the head and working towards the tail (new pages are put in
755: * front of us).
756: * => NOTE: we are allowed to lock the page queues, so the caller
757: * must not be holding the lock on them [e.g. pagedaemon had
758: * better not call us with the queues locked]
759: * => we return TRUE unless we encountered some sort of I/O error
760: * XXXJRT currently never happens, as we never directly initiate
761: * XXXJRT I/O
762: *
763: * comment on "cleaning" object and PG_BUSY pages:
764: * this routine is holding the lock on the object. the only time
765: * that is can run into a PG_BUSY page that it does not own is if
766: * some other process has started I/O on the page (e.g. either
767: * a pagein or a pageout). if the PG_BUSY page is being paged
768: * in, then it can not be dirty (!PG_CLEAN) because no one has
769: * had a change to modify it yet. if the PG_BUSY page is being
770: * paged out then it means that someone else has already started
771: * cleaning the page for us (how nice!). in this case, if we
772: * have syncio specified, then after we make our pass through the
773: * object we need to wait for the other PG_BUSY pages to clear
774: * off (i.e. we need to do an iosync). also note that once a
775: * page is PG_BUSY is must stary in its object until it is un-busyed.
776: * XXXJRT We never actually do this, as we are "flushing" anonymous
777: * XXXJRT memory, which doesn't have persistent backing store.
778: *
779: * note on page traversal:
780: * we can traverse the pages in an object either by going down the
781: * linked list in "uobj->memq", or we can go over the address range
782: * by page doing hash table lookups for each address. depending
783: * on how many pages are in the object it may be cheaper to do one
784: * or the other. we set "by_list" to true if we are using memq.
785: * if the cost of a hash lookup was equal to the cost of the list
786: * traversal we could compare the number of pages in the start->stop
787: * range to the total number of pages in the object. however, it
788: * seems that a hash table lookup is more expensive than the linked
789: * list traversal, so we multiply the number of pages in the
790: * start->stop range by a penalty which we define below.
1.1 mrg 791: */
1.22 thorpej 792:
793: #define UAO_HASH_PENALTY 4 /* XXX: a guess */
794:
1.5 mrg 795: boolean_t
1.22 thorpej 796: uao_flush(uobj, start, stop, flags)
1.5 mrg 797: struct uvm_object *uobj;
1.28 kleink 798: voff_t start, stop;
1.5 mrg 799: int flags;
800: {
1.22 thorpej 801: struct uvm_aobj *aobj = (struct uvm_aobj *) uobj;
802: struct vm_page *pp, *ppnext;
803: boolean_t retval, by_list;
1.28 kleink 804: voff_t curoff;
1.22 thorpej 805: UVMHIST_FUNC("uao_flush"); UVMHIST_CALLED(maphist);
806:
807: curoff = 0; /* XXX: shut up gcc */
808:
809: retval = TRUE; /* default to success */
810:
811: if (flags & PGO_ALLPAGES) {
812: start = 0;
813: stop = aobj->u_pages << PAGE_SHIFT;
814: by_list = TRUE; /* always go by the list */
815: } else {
816: start = trunc_page(start);
817: stop = round_page(stop);
818: if (stop > (aobj->u_pages << PAGE_SHIFT)) {
819: printf("uao_flush: strange, got an out of range "
820: "flush (fixed)\n");
821: stop = aobj->u_pages << PAGE_SHIFT;
822: }
823: by_list = (uobj->uo_npages <=
824: ((stop - start) >> PAGE_SHIFT) * UAO_HASH_PENALTY);
825: }
826:
827: UVMHIST_LOG(maphist,
828: " flush start=0x%lx, stop=0x%x, by_list=%d, flags=0x%x",
829: start, stop, by_list, flags);
1.1 mrg 830:
1.5 mrg 831: /*
1.22 thorpej 832: * Don't need to do any work here if we're not freeing
833: * or deactivating pages.
834: */
835: if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
836: UVMHIST_LOG(maphist,
837: "<- done (no work to do)",0,0,0,0);
838: return (retval);
839: }
840:
1.5 mrg 841: /*
1.22 thorpej 842: * now do it. note: we must update ppnext in the body of loop or we
843: * will get stuck. we need to use ppnext because we may free "pp"
844: * before doing the next loop.
1.21 thorpej 845: */
1.22 thorpej 846:
847: if (by_list) {
848: pp = uobj->memq.tqh_first;
849: } else {
850: curoff = start;
851: pp = uvm_pagelookup(uobj, curoff);
852: }
853:
854: ppnext = NULL; /* XXX: shut up gcc */
855: uvm_lock_pageq(); /* page queues locked */
856:
857: /* locked: both page queues and uobj */
858: for ( ; (by_list && pp != NULL) ||
859: (!by_list && curoff < stop) ; pp = ppnext) {
860: if (by_list) {
861: ppnext = pp->listq.tqe_next;
862:
863: /* range check */
864: if (pp->offset < start || pp->offset >= stop)
865: continue;
866: } else {
867: curoff += PAGE_SIZE;
868: if (curoff < stop)
869: ppnext = uvm_pagelookup(uobj, curoff);
870:
871: /* null check */
872: if (pp == NULL)
873: continue;
874: }
875:
876: switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
877: /*
878: * XXX In these first 3 cases, we always just
879: * XXX deactivate the page. We may want to
880: * XXX handle the different cases more specifically
881: * XXX in the future.
882: */
883: case PGO_CLEANIT|PGO_FREE:
884: case PGO_CLEANIT|PGO_DEACTIVATE:
885: case PGO_DEACTIVATE:
1.25 thorpej 886: deactivate_it:
1.22 thorpej 887: /* skip the page if it's loaned or wired */
888: if (pp->loan_count != 0 ||
889: pp->wire_count != 0)
890: continue;
891:
892: /* zap all mappings for the page. */
1.26 chs 893: pmap_page_protect(pp, VM_PROT_NONE);
1.22 thorpej 894:
895: /* ...and deactivate the page. */
896: uvm_pagedeactivate(pp);
897:
898: continue;
899:
900: case PGO_FREE:
1.25 thorpej 901: /*
902: * If there are multiple references to
903: * the object, just deactivate the page.
904: */
905: if (uobj->uo_refs > 1)
906: goto deactivate_it;
907:
1.22 thorpej 908: /* XXX skip the page if it's loaned or wired */
909: if (pp->loan_count != 0 ||
910: pp->wire_count != 0)
911: continue;
912:
913: /*
914: * mark the page as released if its busy.
915: */
916: if (pp->flags & PG_BUSY) {
917: pp->flags |= PG_RELEASED;
918: continue;
919: }
920:
921: /* zap all mappings for the page. */
1.26 chs 922: pmap_page_protect(pp, VM_PROT_NONE);
1.22 thorpej 923:
924: uao_dropswap(uobj, pp->offset >> PAGE_SHIFT);
925: uvm_pagefree(pp);
926:
927: continue;
928:
929: default:
930: panic("uao_flush: weird flags");
931: }
932: #ifdef DIAGNOSTIC
933: panic("uao_flush: unreachable code");
934: #endif
935: }
936:
937: uvm_unlock_pageq();
938:
939: UVMHIST_LOG(maphist,
940: "<- done, rv=%d",retval,0,0,0);
941: return (retval);
1.1 mrg 942: }
943:
944: /*
945: * uao_get: fetch me a page
946: *
947: * we have three cases:
948: * 1: page is resident -> just return the page.
949: * 2: page is zero-fill -> allocate a new page and zero it.
950: * 3: page is swapped out -> fetch the page from swap.
951: *
952: * cases 1 and 2 can be handled with PGO_LOCKED, case 3 cannot.
953: * so, if the "center" page hits case 3 (or any page, with PGO_ALLPAGES),
954: * then we will need to return VM_PAGER_UNLOCK.
955: *
956: * => prefer map unlocked (not required)
957: * => object must be locked! we will _unlock_ it before starting any I/O.
958: * => flags: PGO_ALLPAGES: get all of the pages
959: * PGO_LOCKED: fault data structures are locked
960: * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
961: * => NOTE: caller must check for released pages!!
962: */
1.5 mrg 963: static int
964: uao_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags)
965: struct uvm_object *uobj;
1.28 kleink 966: voff_t offset;
1.5 mrg 967: struct vm_page **pps;
968: int *npagesp;
969: int centeridx, advice, flags;
970: vm_prot_t access_type;
971: {
972: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.28 kleink 973: voff_t current_offset;
1.5 mrg 974: vm_page_t ptmp;
1.27 chs 975: int lcv, gotpages, maxpages, swslot, rv, pageidx;
1.5 mrg 976: boolean_t done;
977: UVMHIST_FUNC("uao_get"); UVMHIST_CALLED(pdhist);
978:
1.27 chs 979: UVMHIST_LOG(pdhist, "aobj=%p offset=%d, flags=%d",
980: aobj, offset, flags,0);
1.5 mrg 981:
982: /*
983: * get number of pages
984: */
985: maxpages = *npagesp;
986:
987: /*
988: * step 1: handled the case where fault data structures are locked.
989: */
1.1 mrg 990:
1.5 mrg 991: if (flags & PGO_LOCKED) {
992: /*
993: * step 1a: get pages that are already resident. only do
994: * this if the data structures are locked (i.e. the first
995: * time through).
996: */
997:
998: done = TRUE; /* be optimistic */
999: gotpages = 0; /* # of pages we got so far */
1000:
1001: for (lcv = 0, current_offset = offset ; lcv < maxpages ;
1002: lcv++, current_offset += PAGE_SIZE) {
1003: /* do we care about this page? if not, skip it */
1004: if (pps[lcv] == PGO_DONTCARE)
1005: continue;
1006:
1007: ptmp = uvm_pagelookup(uobj, current_offset);
1008:
1009: /*
1.30 thorpej 1010: * if page is new, attempt to allocate the page,
1011: * zero-fill'd.
1.5 mrg 1012: */
1013: if (ptmp == NULL && uao_find_swslot(aobj,
1.15 chs 1014: current_offset >> PAGE_SHIFT) == 0) {
1.5 mrg 1015: ptmp = uvm_pagealloc(uobj, current_offset,
1.30 thorpej 1016: NULL, UVM_PGA_ZERO);
1.5 mrg 1017: if (ptmp) {
1018: /* new page */
1019: ptmp->flags &= ~(PG_BUSY|PG_FAKE);
1020: ptmp->pqflags |= PQ_AOBJ;
1021: UVM_PAGE_OWN(ptmp, NULL);
1022: }
1023: }
1024:
1025: /*
1026: * to be useful must get a non-busy, non-released page
1027: */
1028: if (ptmp == NULL ||
1029: (ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) {
1030: if (lcv == centeridx ||
1031: (flags & PGO_ALLPAGES) != 0)
1032: /* need to do a wait or I/O! */
1033: done = FALSE;
1034: continue;
1035: }
1036:
1037: /*
1038: * useful page: busy/lock it and plug it in our
1039: * result array
1040: */
1041: /* caller must un-busy this page */
1042: ptmp->flags |= PG_BUSY;
1043: UVM_PAGE_OWN(ptmp, "uao_get1");
1044: pps[lcv] = ptmp;
1045: gotpages++;
1046:
1047: } /* "for" lcv loop */
1048:
1049: /*
1050: * step 1b: now we've either done everything needed or we
1051: * to unlock and do some waiting or I/O.
1052: */
1053:
1054: UVMHIST_LOG(pdhist, "<- done (done=%d)", done, 0,0,0);
1055:
1056: *npagesp = gotpages;
1057: if (done)
1058: /* bingo! */
1059: return(VM_PAGER_OK);
1060: else
1061: /* EEK! Need to unlock and I/O */
1062: return(VM_PAGER_UNLOCK);
1.1 mrg 1063: }
1064:
1.5 mrg 1065: /*
1066: * step 2: get non-resident or busy pages.
1067: * object is locked. data structures are unlocked.
1068: */
1069:
1070: for (lcv = 0, current_offset = offset ; lcv < maxpages ;
1071: lcv++, current_offset += PAGE_SIZE) {
1.27 chs 1072:
1.5 mrg 1073: /*
1074: * - skip over pages we've already gotten or don't want
1075: * - skip over pages we don't _have_ to get
1076: */
1.27 chs 1077:
1.5 mrg 1078: if (pps[lcv] != NULL ||
1079: (lcv != centeridx && (flags & PGO_ALLPAGES) == 0))
1080: continue;
1081:
1.27 chs 1082: pageidx = current_offset >> PAGE_SHIFT;
1083:
1.5 mrg 1084: /*
1085: * we have yet to locate the current page (pps[lcv]). we
1086: * first look for a page that is already at the current offset.
1087: * if we find a page, we check to see if it is busy or
1088: * released. if that is the case, then we sleep on the page
1089: * until it is no longer busy or released and repeat the lookup.
1090: * if the page we found is neither busy nor released, then we
1091: * busy it (so we own it) and plug it into pps[lcv]. this
1092: * 'break's the following while loop and indicates we are
1093: * ready to move on to the next page in the "lcv" loop above.
1094: *
1095: * if we exit the while loop with pps[lcv] still set to NULL,
1096: * then it means that we allocated a new busy/fake/clean page
1097: * ptmp in the object and we need to do I/O to fill in the data.
1098: */
1099:
1100: /* top of "pps" while loop */
1101: while (pps[lcv] == NULL) {
1102: /* look for a resident page */
1103: ptmp = uvm_pagelookup(uobj, current_offset);
1104:
1105: /* not resident? allocate one now (if we can) */
1106: if (ptmp == NULL) {
1107:
1108: ptmp = uvm_pagealloc(uobj, current_offset,
1.19 chs 1109: NULL, 0);
1.5 mrg 1110:
1111: /* out of RAM? */
1112: if (ptmp == NULL) {
1113: simple_unlock(&uobj->vmobjlock);
1114: UVMHIST_LOG(pdhist,
1115: "sleeping, ptmp == NULL\n",0,0,0,0);
1116: uvm_wait("uao_getpage");
1117: simple_lock(&uobj->vmobjlock);
1118: /* goto top of pps while loop */
1119: continue;
1120: }
1121:
1122: /*
1123: * safe with PQ's unlocked: because we just
1124: * alloc'd the page
1125: */
1126: ptmp->pqflags |= PQ_AOBJ;
1127:
1128: /*
1129: * got new page ready for I/O. break pps while
1130: * loop. pps[lcv] is still NULL.
1131: */
1132: break;
1133: }
1134:
1135: /* page is there, see if we need to wait on it */
1136: if ((ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) {
1137: ptmp->flags |= PG_WANTED;
1138: UVMHIST_LOG(pdhist,
1139: "sleeping, ptmp->flags 0x%x\n",
1140: ptmp->flags,0,0,0);
1.23 thorpej 1141: UVM_UNLOCK_AND_WAIT(ptmp, &uobj->vmobjlock,
1142: FALSE, "uao_get", 0);
1.5 mrg 1143: simple_lock(&uobj->vmobjlock);
1144: continue; /* goto top of pps while loop */
1145: }
1146:
1147: /*
1148: * if we get here then the page has become resident and
1149: * unbusy between steps 1 and 2. we busy it now (so we
1150: * own it) and set pps[lcv] (so that we exit the while
1151: * loop).
1152: */
1153: /* we own it, caller must un-busy */
1154: ptmp->flags |= PG_BUSY;
1155: UVM_PAGE_OWN(ptmp, "uao_get2");
1156: pps[lcv] = ptmp;
1157: }
1158:
1159: /*
1160: * if we own the valid page at the correct offset, pps[lcv] will
1161: * point to it. nothing more to do except go to the next page.
1162: */
1163: if (pps[lcv])
1164: continue; /* next lcv */
1165:
1166: /*
1167: * we have a "fake/busy/clean" page that we just allocated.
1168: * do the needed "i/o", either reading from swap or zeroing.
1169: */
1.27 chs 1170: swslot = uao_find_swslot(aobj, pageidx);
1.5 mrg 1171:
1172: /*
1173: * just zero the page if there's nothing in swap.
1174: */
1175: if (swslot == 0)
1176: {
1177: /*
1178: * page hasn't existed before, just zero it.
1179: */
1180: uvm_pagezero(ptmp);
1.27 chs 1181: } else {
1.5 mrg 1182: UVMHIST_LOG(pdhist, "pagein from swslot %d",
1183: swslot, 0,0,0);
1184:
1185: /*
1186: * page in the swapped-out page.
1187: * unlock object for i/o, relock when done.
1188: */
1189: simple_unlock(&uobj->vmobjlock);
1190: rv = uvm_swap_get(ptmp, swslot, PGO_SYNCIO);
1191: simple_lock(&uobj->vmobjlock);
1192:
1193: /*
1194: * I/O done. check for errors.
1195: */
1196: if (rv != VM_PAGER_OK)
1197: {
1198: UVMHIST_LOG(pdhist, "<- done (error=%d)",
1199: rv,0,0,0);
1200: if (ptmp->flags & PG_WANTED)
1.24 thorpej 1201: wakeup(ptmp);
1.27 chs 1202:
1203: /*
1204: * remove the swap slot from the aobj
1205: * and mark the aobj as having no real slot.
1206: * don't free the swap slot, thus preventing
1207: * it from being used again.
1208: */
1209: swslot = uao_set_swslot(&aobj->u_obj, pageidx,
1210: SWSLOT_BAD);
1211: uvm_swap_markbad(swslot, 1);
1212:
1.5 mrg 1213: ptmp->flags &= ~(PG_WANTED|PG_BUSY);
1214: UVM_PAGE_OWN(ptmp, NULL);
1215: uvm_lock_pageq();
1216: uvm_pagefree(ptmp);
1217: uvm_unlock_pageq();
1.27 chs 1218:
1.5 mrg 1219: simple_unlock(&uobj->vmobjlock);
1220: return (rv);
1221: }
1222: }
1223:
1224: /*
1225: * we got the page! clear the fake flag (indicates valid
1226: * data now in page) and plug into our result array. note
1227: * that page is still busy.
1228: *
1229: * it is the callers job to:
1230: * => check if the page is released
1231: * => unbusy the page
1232: * => activate the page
1233: */
1234:
1235: ptmp->flags &= ~PG_FAKE; /* data is valid ... */
1.26 chs 1236: pmap_clear_modify(ptmp); /* ... and clean */
1.5 mrg 1237: pps[lcv] = ptmp;
1.1 mrg 1238:
1.5 mrg 1239: } /* lcv loop */
1.1 mrg 1240:
1241: /*
1.5 mrg 1242: * finally, unlock object and return.
1243: */
1.1 mrg 1244:
1245: simple_unlock(&uobj->vmobjlock);
1.5 mrg 1246: UVMHIST_LOG(pdhist, "<- done (OK)",0,0,0,0);
1247: return(VM_PAGER_OK);
1.1 mrg 1248: }
1249:
1250: /*
1251: * uao_releasepg: handle released page in an aobj
1252: *
1253: * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need
1254: * to dispose of.
1255: * => caller must handle PG_WANTED case
1256: * => called with page's object locked, pageq's unlocked
1257: * => returns TRUE if page's object is still alive, FALSE if we
1258: * killed the page's object. if we return TRUE, then we
1259: * return with the object locked.
1260: * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return
1261: * with the page queues locked [for pagedaemon]
1262: * => if (nextpgp == NULL) => we return with page queues unlocked [normal case]
1263: * => we kill the aobj if it is not referenced and we are suppose to
1264: * kill it ("KILLME").
1265: */
1.27 chs 1266: static boolean_t
1267: uao_releasepg(pg, nextpgp)
1.5 mrg 1268: struct vm_page *pg;
1269: struct vm_page **nextpgp; /* OUT */
1.1 mrg 1270: {
1.5 mrg 1271: struct uvm_aobj *aobj = (struct uvm_aobj *) pg->uobject;
1.1 mrg 1272:
1273: #ifdef DIAGNOSTIC
1.5 mrg 1274: if ((pg->flags & PG_RELEASED) == 0)
1275: panic("uao_releasepg: page not released!");
1.1 mrg 1276: #endif
1.27 chs 1277:
1.5 mrg 1278: /*
1279: * dispose of the page [caller handles PG_WANTED] and swap slot.
1280: */
1.26 chs 1281: pmap_page_protect(pg, VM_PROT_NONE);
1.18 chs 1282: uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
1.5 mrg 1283: uvm_lock_pageq();
1284: if (nextpgp)
1285: *nextpgp = pg->pageq.tqe_next; /* next page for daemon */
1286: uvm_pagefree(pg);
1287: if (!nextpgp)
1.27 chs 1288: uvm_unlock_pageq(); /* keep locked for daemon */
1.5 mrg 1289:
1290: /*
1291: * if we're not killing the object, we're done.
1292: */
1293: if ((aobj->u_flags & UAO_FLAG_KILLME) == 0)
1294: return TRUE;
1.1 mrg 1295:
1296: #ifdef DIAGNOSTIC
1.5 mrg 1297: if (aobj->u_obj.uo_refs)
1298: panic("uvm_km_releasepg: kill flag set on referenced object!");
1.1 mrg 1299: #endif
1300:
1.5 mrg 1301: /*
1302: * if there are still pages in the object, we're done for now.
1303: */
1304: if (aobj->u_obj.uo_npages != 0)
1305: return TRUE;
1.1 mrg 1306:
1307: #ifdef DIAGNOSTIC
1.27 chs 1308: if (TAILQ_FIRST(&aobj->u_obj.memq))
1.5 mrg 1309: panic("uvn_releasepg: pages in object with npages == 0");
1.1 mrg 1310: #endif
1311:
1.5 mrg 1312: /*
1313: * finally, free the rest.
1314: */
1315: uao_free(aobj);
1.1 mrg 1316:
1.5 mrg 1317: return FALSE;
1.18 chs 1318: }
1319:
1.27 chs 1320:
1.18 chs 1321: /*
1322: * uao_dropswap: release any swap resources from this aobj page.
1323: *
1324: * => aobj must be locked or have a reference count of 0.
1325: */
1326:
1327: void
1328: uao_dropswap(uobj, pageidx)
1329: struct uvm_object *uobj;
1330: int pageidx;
1331: {
1332: int slot;
1333:
1334: slot = uao_set_swslot(uobj, pageidx, 0);
1335: if (slot) {
1336: uvm_swap_free(slot, 1);
1337: }
1.27 chs 1338: }
1339:
1340:
1341: /*
1342: * page in every page in every aobj that is paged-out to a range of swslots.
1343: *
1344: * => nothing should be locked.
1345: * => returns TRUE if pagein was aborted due to lack of memory.
1346: */
1347: boolean_t
1348: uao_swap_off(startslot, endslot)
1349: int startslot, endslot;
1350: {
1351: struct uvm_aobj *aobj, *nextaobj;
1352:
1353: /*
1354: * walk the list of all aobjs.
1355: */
1356:
1357: restart:
1358: simple_lock(&uao_list_lock);
1359:
1360: for (aobj = LIST_FIRST(&uao_list);
1361: aobj != NULL;
1362: aobj = nextaobj) {
1363: boolean_t rv;
1364:
1365: /*
1366: * try to get the object lock,
1367: * start all over if we fail.
1368: * most of the time we'll get the aobj lock,
1369: * so this should be a rare case.
1370: */
1371: if (!simple_lock_try(&aobj->u_obj.vmobjlock)) {
1372: simple_unlock(&uao_list_lock);
1373: goto restart;
1374: }
1375:
1376: /*
1377: * add a ref to the aobj so it doesn't disappear
1378: * while we're working.
1379: */
1380: uao_reference_locked(&aobj->u_obj);
1381:
1382: /*
1383: * now it's safe to unlock the uao list.
1384: */
1385: simple_unlock(&uao_list_lock);
1386:
1387: /*
1388: * page in any pages in the swslot range.
1389: * if there's an error, abort and return the error.
1390: */
1391: rv = uao_pagein(aobj, startslot, endslot);
1392: if (rv) {
1393: uao_detach_locked(&aobj->u_obj);
1394: return rv;
1395: }
1396:
1397: /*
1398: * we're done with this aobj.
1399: * relock the list and drop our ref on the aobj.
1400: */
1401: simple_lock(&uao_list_lock);
1402: nextaobj = LIST_NEXT(aobj, u_list);
1403: uao_detach_locked(&aobj->u_obj);
1404: }
1405:
1406: /*
1407: * done with traversal, unlock the list
1408: */
1409: simple_unlock(&uao_list_lock);
1410: return FALSE;
1411: }
1412:
1413:
1414: /*
1415: * page in any pages from aobj in the given range.
1416: *
1417: * => aobj must be locked and is returned locked.
1418: * => returns TRUE if pagein was aborted due to lack of memory.
1419: */
1420: static boolean_t
1421: uao_pagein(aobj, startslot, endslot)
1422: struct uvm_aobj *aobj;
1423: int startslot, endslot;
1424: {
1425: boolean_t rv;
1426:
1427: if (UAO_USES_SWHASH(aobj)) {
1428: struct uao_swhash_elt *elt;
1429: int bucket;
1430:
1431: restart:
1432: for (bucket = aobj->u_swhashmask; bucket >= 0; bucket--) {
1433: for (elt = LIST_FIRST(&aobj->u_swhash[bucket]);
1434: elt != NULL;
1435: elt = LIST_NEXT(elt, list)) {
1436: int i;
1437:
1438: for (i = 0; i < UAO_SWHASH_CLUSTER_SIZE; i++) {
1439: int slot = elt->slots[i];
1440:
1441: /*
1442: * if the slot isn't in range, skip it.
1443: */
1444: if (slot < startslot ||
1445: slot >= endslot) {
1446: continue;
1447: }
1448:
1449: /*
1450: * process the page,
1451: * the start over on this object
1452: * since the swhash elt
1453: * may have been freed.
1454: */
1455: rv = uao_pagein_page(aobj,
1456: UAO_SWHASH_ELT_PAGEIDX_BASE(elt) + i);
1457: if (rv) {
1458: return rv;
1459: }
1460: goto restart;
1461: }
1462: }
1463: }
1464: } else {
1465: int i;
1466:
1467: for (i = 0; i < aobj->u_pages; i++) {
1468: int slot = aobj->u_swslots[i];
1469:
1470: /*
1471: * if the slot isn't in range, skip it
1472: */
1473: if (slot < startslot || slot >= endslot) {
1474: continue;
1475: }
1476:
1477: /*
1478: * process the page.
1479: */
1480: rv = uao_pagein_page(aobj, i);
1481: if (rv) {
1482: return rv;
1483: }
1484: }
1485: }
1486:
1487: return FALSE;
1488: }
1489:
1490: /*
1491: * page in a page from an aobj. used for swap_off.
1492: * returns TRUE if pagein was aborted due to lack of memory.
1493: *
1494: * => aobj must be locked and is returned locked.
1495: */
1496: static boolean_t
1497: uao_pagein_page(aobj, pageidx)
1498: struct uvm_aobj *aobj;
1499: int pageidx;
1500: {
1501: struct vm_page *pg;
1502: int rv, slot, npages;
1503: UVMHIST_FUNC("uao_pagein_page"); UVMHIST_CALLED(pdhist);
1504:
1505: pg = NULL;
1506: npages = 1;
1507: /* locked: aobj */
1508: rv = uao_get(&aobj->u_obj, pageidx << PAGE_SHIFT,
1509: &pg, &npages, 0, VM_PROT_READ|VM_PROT_WRITE, 0, 0);
1510: /* unlocked: aobj */
1511:
1512: /*
1513: * relock and finish up.
1514: */
1515: simple_lock(&aobj->u_obj.vmobjlock);
1516:
1517: switch (rv) {
1518: case VM_PAGER_OK:
1519: break;
1520:
1521: case VM_PAGER_ERROR:
1522: case VM_PAGER_REFAULT:
1523: /*
1524: * nothing more to do on errors.
1525: * VM_PAGER_REFAULT can only mean that the anon was freed,
1526: * so again there's nothing to do.
1527: */
1528: return FALSE;
1529:
1530: #ifdef DIAGNOSTIC
1531: default:
1532: panic("uao_pagein_page: uao_get -> %d\n", rv);
1533: #endif
1534: }
1535:
1536: #ifdef DIAGNOSTIC
1537: /*
1538: * this should never happen, since we have a reference on the aobj.
1539: */
1540: if (pg->flags & PG_RELEASED) {
1541: panic("uao_pagein_page: found PG_RELEASED page?\n");
1542: }
1543: #endif
1544:
1545: /*
1546: * ok, we've got the page now.
1547: * mark it as dirty, clear its swslot and un-busy it.
1548: */
1549: slot = uao_set_swslot(&aobj->u_obj, pageidx, 0);
1550: uvm_swap_free(slot, 1);
1551: pg->flags &= ~(PG_BUSY|PG_CLEAN|PG_FAKE);
1552: UVM_PAGE_OWN(pg, NULL);
1553:
1554: /*
1555: * deactivate the page (to put it on a page queue).
1556: */
1557: pmap_clear_reference(pg);
1558: pmap_page_protect(pg, VM_PROT_NONE);
1559: uvm_lock_pageq();
1560: uvm_pagedeactivate(pg);
1561: uvm_unlock_pageq();
1562:
1563: return FALSE;
1.1 mrg 1564: }
CVSweb <webmaster@jp.NetBSD.org>