Annotation of src/sys/uvm/uvm_aobj.c, Revision 1.68
1.68 ! thorpej 1: /* $NetBSD: uvm_aobj.c,v 1.67 2005/06/27 02:19:48 thorpej Exp $ */
1.6 mrg 2:
1.7 chs 3: /*
4: * Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
5: * Washington University.
6: * All rights reserved.
7: *
8: * Redistribution and use in source and binary forms, with or without
9: * modification, are permitted provided that the following conditions
10: * are met:
11: * 1. Redistributions of source code must retain the above copyright
12: * notice, this list of conditions and the following disclaimer.
13: * 2. Redistributions in binary form must reproduce the above copyright
14: * notice, this list of conditions and the following disclaimer in the
15: * documentation and/or other materials provided with the distribution.
16: * 3. All advertising materials mentioning features or use of this software
17: * must display the following acknowledgement:
18: * This product includes software developed by Charles D. Cranor and
19: * Washington University.
20: * 4. The name of the author may not be used to endorse or promote products
21: * derived from this software without specific prior written permission.
22: *
23: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33: *
1.4 mrg 34: * from: Id: uvm_aobj.c,v 1.1.2.5 1998/02/06 05:14:38 chs Exp
35: */
1.7 chs 36: /*
37: * uvm_aobj.c: anonymous memory uvm_object pager
38: *
39: * author: Chuck Silvers <chuq@chuq.com>
40: * started: Jan-1998
41: *
42: * - design mostly from Chuck Cranor
43: */
1.49 lukem 44:
45: #include <sys/cdefs.h>
1.68 ! thorpej 46: __KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.67 2005/06/27 02:19:48 thorpej Exp $");
1.7 chs 47:
48: #include "opt_uvmhist.h"
1.1 mrg 49:
50: #include <sys/param.h>
51: #include <sys/systm.h>
52: #include <sys/proc.h>
53: #include <sys/malloc.h>
1.37 chs 54: #include <sys/kernel.h>
1.12 thorpej 55: #include <sys/pool.h>
1.27 chs 56: #include <sys/kernel.h>
1.1 mrg 57:
58: #include <uvm/uvm.h>
59:
60: /*
61: * an aobj manages anonymous-memory backed uvm_objects. in addition
62: * to keeping the list of resident pages, it also keeps a list of
63: * allocated swap blocks. depending on the size of the aobj this list
64: * of allocated swap blocks is either stored in an array (small objects)
65: * or in a hash table (large objects).
66: */
67:
68: /*
69: * local structures
70: */
71:
72: /*
73: * for hash tables, we break the address space of the aobj into blocks
74: * of UAO_SWHASH_CLUSTER_SIZE pages. we require the cluster size to
75: * be a power of two.
76: */
77:
78: #define UAO_SWHASH_CLUSTER_SHIFT 4
79: #define UAO_SWHASH_CLUSTER_SIZE (1 << UAO_SWHASH_CLUSTER_SHIFT)
80:
81: /* get the "tag" for this page index */
82: #define UAO_SWHASH_ELT_TAG(PAGEIDX) \
83: ((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT)
84:
85: /* given an ELT and a page index, find the swap slot */
86: #define UAO_SWHASH_ELT_PAGESLOT(ELT, PAGEIDX) \
87: ((ELT)->slots[(PAGEIDX) & (UAO_SWHASH_CLUSTER_SIZE - 1)])
88:
89: /* given an ELT, return its pageidx base */
90: #define UAO_SWHASH_ELT_PAGEIDX_BASE(ELT) \
91: ((ELT)->tag << UAO_SWHASH_CLUSTER_SHIFT)
92:
93: /*
94: * the swhash hash function
95: */
1.46 chs 96:
1.1 mrg 97: #define UAO_SWHASH_HASH(AOBJ, PAGEIDX) \
98: (&(AOBJ)->u_swhash[(((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT) \
99: & (AOBJ)->u_swhashmask)])
100:
101: /*
102: * the swhash threshhold determines if we will use an array or a
103: * hash table to store the list of allocated swap blocks.
104: */
105:
106: #define UAO_SWHASH_THRESHOLD (UAO_SWHASH_CLUSTER_SIZE * 4)
107: #define UAO_USES_SWHASH(AOBJ) \
108: ((AOBJ)->u_pages > UAO_SWHASH_THRESHOLD) /* use hash? */
109:
110: /*
1.3 chs 111: * the number of buckets in a swhash, with an upper bound
1.1 mrg 112: */
1.46 chs 113:
1.1 mrg 114: #define UAO_SWHASH_MAXBUCKETS 256
115: #define UAO_SWHASH_BUCKETS(AOBJ) \
1.46 chs 116: (MIN((AOBJ)->u_pages >> UAO_SWHASH_CLUSTER_SHIFT, \
1.1 mrg 117: UAO_SWHASH_MAXBUCKETS))
118:
119:
120: /*
121: * uao_swhash_elt: when a hash table is being used, this structure defines
122: * the format of an entry in the bucket list.
123: */
124:
125: struct uao_swhash_elt {
1.5 mrg 126: LIST_ENTRY(uao_swhash_elt) list; /* the hash list */
1.28 kleink 127: voff_t tag; /* our 'tag' */
1.5 mrg 128: int count; /* our number of active slots */
129: int slots[UAO_SWHASH_CLUSTER_SIZE]; /* the slots */
1.1 mrg 130: };
131:
132: /*
133: * uao_swhash: the swap hash table structure
134: */
135:
136: LIST_HEAD(uao_swhash, uao_swhash_elt);
137:
1.12 thorpej 138: /*
139: * uao_swhash_elt_pool: pool of uao_swhash_elt structures
1.64 simonb 140: * NOTE: Pages for this pool must not come from a pageable kernel map!
1.12 thorpej 141: */
1.64 simonb 142: POOL_INIT(uao_swhash_elt_pool, sizeof(struct uao_swhash_elt), 0, 0, 0,
143: "uaoeltpl", NULL);
1.1 mrg 144:
145: /*
146: * uvm_aobj: the actual anon-backed uvm_object
147: *
148: * => the uvm_object is at the top of the structure, this allows
1.46 chs 149: * (struct uvm_aobj *) == (struct uvm_object *)
1.1 mrg 150: * => only one of u_swslots and u_swhash is used in any given aobj
151: */
152:
153: struct uvm_aobj {
1.5 mrg 154: struct uvm_object u_obj; /* has: lock, pgops, memq, #pages, #refs */
1.11 drochner 155: int u_pages; /* number of pages in entire object */
1.5 mrg 156: int u_flags; /* the flags (see uvm_aobj.h) */
157: int *u_swslots; /* array of offset->swapslot mappings */
158: /*
159: * hashtable of offset->swapslot mappings
160: * (u_swhash is an array of bucket heads)
161: */
162: struct uao_swhash *u_swhash;
163: u_long u_swhashmask; /* mask for hashtable */
164: LIST_ENTRY(uvm_aobj) u_list; /* global list of aobjs */
1.1 mrg 165: };
166:
167: /*
1.12 thorpej 168: * uvm_aobj_pool: pool of uvm_aobj structures
169: */
1.64 simonb 170: POOL_INIT(uvm_aobj_pool, sizeof(struct uvm_aobj), 0, 0, 0, "aobjpl",
171: &pool_allocator_nointr);
1.54 thorpej 172:
173: MALLOC_DEFINE(M_UVMAOBJ, "UVM aobj", "UVM aobj and related structures");
1.12 thorpej 174:
175: /*
1.1 mrg 176: * local functions
177: */
178:
1.46 chs 179: static struct uao_swhash_elt *uao_find_swhash_elt
1.62 junyoung 180: (struct uvm_aobj *, int, boolean_t);
1.46 chs 181:
1.62 junyoung 182: static void uao_free(struct uvm_aobj *);
183: static int uao_get(struct uvm_object *, voff_t, struct vm_page **,
184: int *, int, vm_prot_t, int, int);
185: static boolean_t uao_put(struct uvm_object *, voff_t, voff_t, int);
186: static boolean_t uao_pagein(struct uvm_aobj *, int, int);
187: static boolean_t uao_pagein_page(struct uvm_aobj *, int);
1.1 mrg 188:
189: /*
190: * aobj_pager
1.41 chs 191: *
1.1 mrg 192: * note that some functions (e.g. put) are handled elsewhere
193: */
194:
195: struct uvm_pagerops aobj_pager = {
1.27 chs 196: NULL, /* init */
1.5 mrg 197: uao_reference, /* reference */
198: uao_detach, /* detach */
199: NULL, /* fault */
200: uao_get, /* get */
1.46 chs 201: uao_put, /* flush */
1.1 mrg 202: };
203:
204: /*
205: * uao_list: global list of active aobjs, locked by uao_list_lock
206: */
207:
208: static LIST_HEAD(aobjlist, uvm_aobj) uao_list;
1.42 chs 209: static struct simplelock uao_list_lock;
1.1 mrg 210:
211: /*
212: * functions
213: */
214:
215: /*
216: * hash table/array related functions
217: */
218:
219: /*
220: * uao_find_swhash_elt: find (or create) a hash table entry for a page
221: * offset.
222: *
223: * => the object should be locked by the caller
224: */
225:
1.5 mrg 226: static struct uao_swhash_elt *
1.67 thorpej 227: uao_find_swhash_elt(struct uvm_aobj *aobj, int pageidx, boolean_t create)
1.5 mrg 228: {
229: struct uao_swhash *swhash;
230: struct uao_swhash_elt *elt;
1.28 kleink 231: voff_t page_tag;
1.1 mrg 232:
1.45 chs 233: swhash = UAO_SWHASH_HASH(aobj, pageidx);
234: page_tag = UAO_SWHASH_ELT_TAG(pageidx);
1.1 mrg 235:
1.5 mrg 236: /*
237: * now search the bucket for the requested tag
238: */
1.45 chs 239:
1.37 chs 240: LIST_FOREACH(elt, swhash, list) {
1.45 chs 241: if (elt->tag == page_tag) {
242: return elt;
243: }
1.5 mrg 244: }
1.45 chs 245: if (!create) {
1.5 mrg 246: return NULL;
1.45 chs 247: }
1.5 mrg 248:
249: /*
1.12 thorpej 250: * allocate a new entry for the bucket and init/insert it in
1.5 mrg 251: */
1.45 chs 252:
253: elt = pool_get(&uao_swhash_elt_pool, PR_NOWAIT);
254: if (elt == NULL) {
255: return NULL;
256: }
1.5 mrg 257: LIST_INSERT_HEAD(swhash, elt, list);
258: elt->tag = page_tag;
259: elt->count = 0;
1.9 perry 260: memset(elt->slots, 0, sizeof(elt->slots));
1.45 chs 261: return elt;
1.1 mrg 262: }
263:
264: /*
265: * uao_find_swslot: find the swap slot number for an aobj/pageidx
266: *
1.41 chs 267: * => object must be locked by caller
1.1 mrg 268: */
1.46 chs 269:
270: int
1.67 thorpej 271: uao_find_swslot(struct uvm_object *uobj, int pageidx)
1.1 mrg 272: {
1.46 chs 273: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
274: struct uao_swhash_elt *elt;
1.1 mrg 275:
1.5 mrg 276: /*
277: * if noswap flag is set, then we never return a slot
278: */
1.1 mrg 279:
1.5 mrg 280: if (aobj->u_flags & UAO_FLAG_NOSWAP)
281: return(0);
1.1 mrg 282:
1.5 mrg 283: /*
284: * if hashing, look in hash table.
285: */
1.1 mrg 286:
1.5 mrg 287: if (UAO_USES_SWHASH(aobj)) {
1.46 chs 288: elt = uao_find_swhash_elt(aobj, pageidx, FALSE);
1.5 mrg 289: if (elt)
290: return(UAO_SWHASH_ELT_PAGESLOT(elt, pageidx));
291: else
1.31 thorpej 292: return(0);
1.5 mrg 293: }
1.1 mrg 294:
1.41 chs 295: /*
1.5 mrg 296: * otherwise, look in the array
297: */
1.46 chs 298:
1.5 mrg 299: return(aobj->u_swslots[pageidx]);
1.1 mrg 300: }
301:
302: /*
303: * uao_set_swslot: set the swap slot for a page in an aobj.
304: *
305: * => setting a slot to zero frees the slot
306: * => object must be locked by caller
1.45 chs 307: * => we return the old slot number, or -1 if we failed to allocate
308: * memory to record the new slot number
1.1 mrg 309: */
1.46 chs 310:
1.5 mrg 311: int
1.67 thorpej 312: uao_set_swslot(struct uvm_object *uobj, int pageidx, int slot)
1.5 mrg 313: {
314: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.45 chs 315: struct uao_swhash_elt *elt;
1.5 mrg 316: int oldslot;
317: UVMHIST_FUNC("uao_set_swslot"); UVMHIST_CALLED(pdhist);
318: UVMHIST_LOG(pdhist, "aobj %p pageidx %d slot %d",
319: aobj, pageidx, slot, 0);
1.1 mrg 320:
1.5 mrg 321: /*
1.46 chs 322: * if noswap flag is set, then we can't set a non-zero slot.
1.5 mrg 323: */
1.1 mrg 324:
1.5 mrg 325: if (aobj->u_flags & UAO_FLAG_NOSWAP) {
326: if (slot == 0)
1.46 chs 327: return(0);
1.1 mrg 328:
1.5 mrg 329: printf("uao_set_swslot: uobj = %p\n", uobj);
1.46 chs 330: panic("uao_set_swslot: NOSWAP object");
1.5 mrg 331: }
1.1 mrg 332:
1.5 mrg 333: /*
334: * are we using a hash table? if so, add it in the hash.
335: */
1.1 mrg 336:
1.5 mrg 337: if (UAO_USES_SWHASH(aobj)) {
1.39 chs 338:
1.12 thorpej 339: /*
340: * Avoid allocating an entry just to free it again if
341: * the page had not swap slot in the first place, and
342: * we are freeing.
343: */
1.39 chs 344:
1.46 chs 345: elt = uao_find_swhash_elt(aobj, pageidx, slot != 0);
1.12 thorpej 346: if (elt == NULL) {
1.45 chs 347: return slot ? -1 : 0;
1.12 thorpej 348: }
1.5 mrg 349:
350: oldslot = UAO_SWHASH_ELT_PAGESLOT(elt, pageidx);
351: UAO_SWHASH_ELT_PAGESLOT(elt, pageidx) = slot;
352:
353: /*
354: * now adjust the elt's reference counter and free it if we've
355: * dropped it to zero.
356: */
357:
358: if (slot) {
359: if (oldslot == 0)
360: elt->count++;
1.45 chs 361: } else {
362: if (oldslot)
1.5 mrg 363: elt->count--;
364:
365: if (elt->count == 0) {
366: LIST_REMOVE(elt, list);
1.12 thorpej 367: pool_put(&uao_swhash_elt_pool, elt);
1.5 mrg 368: }
369: }
1.41 chs 370: } else {
1.5 mrg 371: /* we are using an array */
372: oldslot = aobj->u_swslots[pageidx];
373: aobj->u_swslots[pageidx] = slot;
374: }
375: return (oldslot);
1.1 mrg 376: }
377:
378: /*
379: * end of hash/array functions
380: */
381:
382: /*
383: * uao_free: free all resources held by an aobj, and then free the aobj
384: *
385: * => the aobj should be dead
386: */
1.46 chs 387:
1.1 mrg 388: static void
1.67 thorpej 389: uao_free(struct uvm_aobj *aobj)
1.1 mrg 390: {
1.46 chs 391: int swpgonlydelta = 0;
1.1 mrg 392:
1.27 chs 393: simple_unlock(&aobj->u_obj.vmobjlock);
1.5 mrg 394: if (UAO_USES_SWHASH(aobj)) {
395: int i, hashbuckets = aobj->u_swhashmask + 1;
1.1 mrg 396:
1.5 mrg 397: /*
398: * free the swslots from each hash bucket,
399: * then the hash bucket, and finally the hash table itself.
400: */
1.46 chs 401:
1.5 mrg 402: for (i = 0; i < hashbuckets; i++) {
403: struct uao_swhash_elt *elt, *next;
404:
1.27 chs 405: for (elt = LIST_FIRST(&aobj->u_swhash[i]);
406: elt != NULL;
407: elt = next) {
1.5 mrg 408: int j;
409:
1.27 chs 410: for (j = 0; j < UAO_SWHASH_CLUSTER_SIZE; j++) {
1.5 mrg 411: int slot = elt->slots[j];
412:
1.60 pk 413: if (slot > 0) {
1.61 drochner 414: uvm_swap_free(slot, 1);
415: swpgonlydelta++;
1.37 chs 416: }
1.5 mrg 417: }
418:
1.27 chs 419: next = LIST_NEXT(elt, list);
1.12 thorpej 420: pool_put(&uao_swhash_elt_pool, elt);
1.5 mrg 421: }
422: }
1.34 thorpej 423: free(aobj->u_swhash, M_UVMAOBJ);
1.5 mrg 424: } else {
425: int i;
426:
427: /*
428: * free the array
429: */
430:
1.27 chs 431: for (i = 0; i < aobj->u_pages; i++) {
1.5 mrg 432: int slot = aobj->u_swslots[i];
433:
1.60 pk 434: if (slot > 0) {
1.5 mrg 435: uvm_swap_free(slot, 1);
1.46 chs 436: swpgonlydelta++;
1.18 chs 437: }
1.5 mrg 438: }
1.34 thorpej 439: free(aobj->u_swslots, M_UVMAOBJ);
1.1 mrg 440: }
441:
1.5 mrg 442: /*
443: * finally free the aobj itself
444: */
1.46 chs 445:
1.12 thorpej 446: pool_put(&uvm_aobj_pool, aobj);
1.46 chs 447:
448: /*
449: * adjust the counter of pages only in swap for all
450: * the swap slots we've freed.
451: */
452:
1.48 chs 453: if (swpgonlydelta > 0) {
454: simple_lock(&uvm.swap_data_lock);
455: KASSERT(uvmexp.swpgonly >= swpgonlydelta);
456: uvmexp.swpgonly -= swpgonlydelta;
457: simple_unlock(&uvm.swap_data_lock);
458: }
1.1 mrg 459: }
460:
461: /*
462: * pager functions
463: */
464:
465: /*
466: * uao_create: create an aobj of the given size and return its uvm_object.
467: *
468: * => for normal use, flags are always zero
469: * => for the kernel object, the flags are:
470: * UAO_FLAG_KERNOBJ - allocate the kernel object (can only happen once)
471: * UAO_FLAG_KERNSWAP - enable swapping of kernel object (" ")
472: */
1.46 chs 473:
1.5 mrg 474: struct uvm_object *
1.67 thorpej 475: uao_create(vsize_t size, int flags)
1.5 mrg 476: {
1.46 chs 477: static struct uvm_aobj kernel_object_store;
478: static int kobj_alloced = 0;
1.15 chs 479: int pages = round_page(size) >> PAGE_SHIFT;
1.5 mrg 480: struct uvm_aobj *aobj;
1.66 yamt 481: int refs;
1.1 mrg 482:
1.5 mrg 483: /*
1.27 chs 484: * malloc a new aobj unless we are asked for the kernel object
485: */
1.5 mrg 486:
1.46 chs 487: if (flags & UAO_FLAG_KERNOBJ) {
488: KASSERT(!kobj_alloced);
1.5 mrg 489: aobj = &kernel_object_store;
490: aobj->u_pages = pages;
1.46 chs 491: aobj->u_flags = UAO_FLAG_NOSWAP;
1.66 yamt 492: refs = UVM_OBJ_KERN;
1.5 mrg 493: kobj_alloced = UAO_FLAG_KERNOBJ;
494: } else if (flags & UAO_FLAG_KERNSWAP) {
1.46 chs 495: KASSERT(kobj_alloced == UAO_FLAG_KERNOBJ);
1.5 mrg 496: aobj = &kernel_object_store;
497: kobj_alloced = UAO_FLAG_KERNSWAP;
1.66 yamt 498: refs = 0xdeadbeaf; /* XXX: gcc */
1.46 chs 499: } else {
1.12 thorpej 500: aobj = pool_get(&uvm_aobj_pool, PR_WAITOK);
1.5 mrg 501: aobj->u_pages = pages;
1.46 chs 502: aobj->u_flags = 0;
1.66 yamt 503: refs = 1;
1.5 mrg 504: }
1.1 mrg 505:
1.5 mrg 506: /*
507: * allocate hash/array if necessary
508: *
509: * note: in the KERNSWAP case no need to worry about locking since
510: * we are still booting we should be the only thread around.
511: */
1.46 chs 512:
1.5 mrg 513: if (flags == 0 || (flags & UAO_FLAG_KERNSWAP) != 0) {
514: int mflags = (flags & UAO_FLAG_KERNSWAP) != 0 ?
515: M_NOWAIT : M_WAITOK;
516:
517: /* allocate hash table or array depending on object size */
1.27 chs 518: if (UAO_USES_SWHASH(aobj)) {
1.5 mrg 519: aobj->u_swhash = hashinit(UAO_SWHASH_BUCKETS(aobj),
1.35 ad 520: HASH_LIST, M_UVMAOBJ, mflags, &aobj->u_swhashmask);
1.5 mrg 521: if (aobj->u_swhash == NULL)
522: panic("uao_create: hashinit swhash failed");
523: } else {
1.34 thorpej 524: aobj->u_swslots = malloc(pages * sizeof(int),
1.5 mrg 525: M_UVMAOBJ, mflags);
526: if (aobj->u_swslots == NULL)
527: panic("uao_create: malloc swslots failed");
1.9 perry 528: memset(aobj->u_swslots, 0, pages * sizeof(int));
1.5 mrg 529: }
530:
531: if (flags) {
532: aobj->u_flags &= ~UAO_FLAG_NOSWAP; /* clear noswap */
533: return(&aobj->u_obj);
534: }
535: }
536:
537: /*
538: * init aobj fields
539: */
1.46 chs 540:
1.66 yamt 541: UVM_OBJ_INIT(&aobj->u_obj, &aobj_pager, refs);
1.1 mrg 542:
1.5 mrg 543: /*
544: * now that aobj is ready, add it to the global list
545: */
1.46 chs 546:
1.5 mrg 547: simple_lock(&uao_list_lock);
548: LIST_INSERT_HEAD(&uao_list, aobj, u_list);
549: simple_unlock(&uao_list_lock);
550: return(&aobj->u_obj);
1.1 mrg 551: }
552:
553:
554:
555: /*
556: * uao_init: set up aobj pager subsystem
557: *
558: * => called at boot time from uvm_pager_init()
559: */
1.46 chs 560:
1.27 chs 561: void
1.46 chs 562: uao_init(void)
1.5 mrg 563: {
1.12 thorpej 564: static int uao_initialized;
565:
566: if (uao_initialized)
567: return;
568: uao_initialized = TRUE;
1.5 mrg 569: LIST_INIT(&uao_list);
570: simple_lock_init(&uao_list_lock);
1.1 mrg 571: }
572:
573: /*
574: * uao_reference: add a ref to an aobj
575: *
1.27 chs 576: * => aobj must be unlocked
577: * => just lock it and call the locked version
1.1 mrg 578: */
1.46 chs 579:
1.5 mrg 580: void
1.67 thorpej 581: uao_reference(struct uvm_object *uobj)
1.1 mrg 582: {
1.27 chs 583: simple_lock(&uobj->vmobjlock);
584: uao_reference_locked(uobj);
585: simple_unlock(&uobj->vmobjlock);
586: }
587:
588: /*
589: * uao_reference_locked: add a ref to an aobj that is already locked
590: *
591: * => aobj must be locked
592: * this needs to be separate from the normal routine
593: * since sometimes we need to add a reference to an aobj when
594: * it's already locked.
595: */
1.46 chs 596:
1.27 chs 597: void
1.67 thorpej 598: uao_reference_locked(struct uvm_object *uobj)
1.27 chs 599: {
1.5 mrg 600: UVMHIST_FUNC("uao_reference"); UVMHIST_CALLED(maphist);
1.1 mrg 601:
1.5 mrg 602: /*
603: * kernel_object already has plenty of references, leave it alone.
604: */
1.1 mrg 605:
1.20 thorpej 606: if (UVM_OBJ_IS_KERN_OBJECT(uobj))
1.5 mrg 607: return;
1.1 mrg 608:
1.46 chs 609: uobj->uo_refs++;
1.41 chs 610: UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)",
1.27 chs 611: uobj, uobj->uo_refs,0,0);
1.1 mrg 612: }
613:
614: /*
615: * uao_detach: drop a reference to an aobj
616: *
1.27 chs 617: * => aobj must be unlocked
618: * => just lock it and call the locked version
1.1 mrg 619: */
1.46 chs 620:
1.5 mrg 621: void
1.67 thorpej 622: uao_detach(struct uvm_object *uobj)
1.5 mrg 623: {
1.27 chs 624: simple_lock(&uobj->vmobjlock);
625: uao_detach_locked(uobj);
626: }
627:
628: /*
629: * uao_detach_locked: drop a reference to an aobj
630: *
631: * => aobj must be locked, and is unlocked (or freed) upon return.
632: * this needs to be separate from the normal routine
633: * since sometimes we need to detach from an aobj when
634: * it's already locked.
635: */
1.46 chs 636:
1.27 chs 637: void
1.67 thorpej 638: uao_detach_locked(struct uvm_object *uobj)
1.27 chs 639: {
1.5 mrg 640: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.46 chs 641: struct vm_page *pg;
1.5 mrg 642: UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist);
1.1 mrg 643:
1.5 mrg 644: /*
645: * detaching from kernel_object is a noop.
646: */
1.46 chs 647:
1.27 chs 648: if (UVM_OBJ_IS_KERN_OBJECT(uobj)) {
649: simple_unlock(&uobj->vmobjlock);
1.5 mrg 650: return;
1.27 chs 651: }
1.5 mrg 652:
653: UVMHIST_LOG(maphist," (uobj=0x%x) ref=%d", uobj,uobj->uo_refs,0,0);
1.46 chs 654: uobj->uo_refs--;
655: if (uobj->uo_refs) {
1.5 mrg 656: simple_unlock(&uobj->vmobjlock);
657: UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
658: return;
659: }
660:
661: /*
662: * remove the aobj from the global list.
663: */
1.46 chs 664:
1.5 mrg 665: simple_lock(&uao_list_lock);
666: LIST_REMOVE(aobj, u_list);
667: simple_unlock(&uao_list_lock);
668:
669: /*
1.46 chs 670: * free all the pages left in the aobj. for each page,
671: * when the page is no longer busy (and thus after any disk i/o that
672: * it's involved in is complete), release any swap resources and
673: * free the page itself.
1.5 mrg 674: */
1.46 chs 675:
676: uvm_lock_pageq();
677: while ((pg = TAILQ_FIRST(&uobj->memq)) != NULL) {
678: pmap_page_protect(pg, VM_PROT_NONE);
1.5 mrg 679: if (pg->flags & PG_BUSY) {
1.46 chs 680: pg->flags |= PG_WANTED;
681: uvm_unlock_pageq();
682: UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, FALSE,
683: "uao_det", 0);
684: simple_lock(&uobj->vmobjlock);
685: uvm_lock_pageq();
1.5 mrg 686: continue;
687: }
1.18 chs 688: uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
1.5 mrg 689: uvm_pagefree(pg);
690: }
1.46 chs 691: uvm_unlock_pageq();
1.1 mrg 692:
1.5 mrg 693: /*
1.46 chs 694: * finally, free the aobj itself.
1.5 mrg 695: */
1.1 mrg 696:
1.5 mrg 697: uao_free(aobj);
698: }
1.1 mrg 699:
700: /*
1.46 chs 701: * uao_put: flush pages out of a uvm object
1.22 thorpej 702: *
703: * => object should be locked by caller. we may _unlock_ the object
704: * if (and only if) we need to clean a page (PGO_CLEANIT).
705: * XXXJRT Currently, however, we don't. In the case of cleaning
706: * XXXJRT a page, we simply just deactivate it. Should probably
707: * XXXJRT handle this better, in the future (although "flushing"
708: * XXXJRT anonymous memory isn't terribly important).
709: * => if PGO_CLEANIT is not set, then we will neither unlock the object
710: * or block.
711: * => if PGO_ALLPAGE is set, then all pages in the object are valid targets
712: * for flushing.
713: * => NOTE: we rely on the fact that the object's memq is a TAILQ and
714: * that new pages are inserted on the tail end of the list. thus,
715: * we can make a complete pass through the object in one go by starting
716: * at the head and working towards the tail (new pages are put in
717: * front of us).
718: * => NOTE: we are allowed to lock the page queues, so the caller
719: * must not be holding the lock on them [e.g. pagedaemon had
720: * better not call us with the queues locked]
721: * => we return TRUE unless we encountered some sort of I/O error
722: * XXXJRT currently never happens, as we never directly initiate
723: * XXXJRT I/O
724: *
725: * note on page traversal:
726: * we can traverse the pages in an object either by going down the
727: * linked list in "uobj->memq", or we can go over the address range
728: * by page doing hash table lookups for each address. depending
729: * on how many pages are in the object it may be cheaper to do one
730: * or the other. we set "by_list" to true if we are using memq.
731: * if the cost of a hash lookup was equal to the cost of the list
732: * traversal we could compare the number of pages in the start->stop
733: * range to the total number of pages in the object. however, it
734: * seems that a hash table lookup is more expensive than the linked
735: * list traversal, so we multiply the number of pages in the
736: * start->stop range by a penalty which we define below.
1.1 mrg 737: */
1.22 thorpej 738:
1.68 ! thorpej 739: static int
1.67 thorpej 740: uao_put(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
1.5 mrg 741: {
1.46 chs 742: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.51 enami 743: struct vm_page *pg, *nextpg, curmp, endmp;
1.46 chs 744: boolean_t by_list;
1.28 kleink 745: voff_t curoff;
1.46 chs 746: UVMHIST_FUNC("uao_put"); UVMHIST_CALLED(maphist);
1.22 thorpej 747:
1.46 chs 748: curoff = 0;
1.22 thorpej 749: if (flags & PGO_ALLPAGES) {
750: start = 0;
751: stop = aobj->u_pages << PAGE_SHIFT;
752: by_list = TRUE; /* always go by the list */
753: } else {
754: start = trunc_page(start);
755: stop = round_page(stop);
756: if (stop > (aobj->u_pages << PAGE_SHIFT)) {
757: printf("uao_flush: strange, got an out of range "
758: "flush (fixed)\n");
759: stop = aobj->u_pages << PAGE_SHIFT;
760: }
761: by_list = (uobj->uo_npages <=
1.46 chs 762: ((stop - start) >> PAGE_SHIFT) * UVM_PAGE_HASH_PENALTY);
1.22 thorpej 763: }
764: UVMHIST_LOG(maphist,
765: " flush start=0x%lx, stop=0x%x, by_list=%d, flags=0x%x",
766: start, stop, by_list, flags);
1.1 mrg 767:
1.5 mrg 768: /*
1.22 thorpej 769: * Don't need to do any work here if we're not freeing
770: * or deactivating pages.
771: */
1.46 chs 772:
1.22 thorpej 773: if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
1.46 chs 774: simple_unlock(&uobj->vmobjlock);
775: return 0;
1.22 thorpej 776: }
777:
1.5 mrg 778: /*
1.51 enami 779: * Initialize the marker pages. See the comment in
780: * genfs_putpages() also.
781: */
782:
783: curmp.uobject = uobj;
784: curmp.offset = (voff_t)-1;
785: curmp.flags = PG_BUSY;
786: endmp.uobject = uobj;
787: endmp.offset = (voff_t)-1;
788: endmp.flags = PG_BUSY;
789:
790: /*
1.46 chs 791: * now do it. note: we must update nextpg in the body of loop or we
1.51 enami 792: * will get stuck. we need to use nextpg if we'll traverse the list
793: * because we may free "pg" before doing the next loop.
1.21 thorpej 794: */
1.22 thorpej 795:
796: if (by_list) {
1.51 enami 797: TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq);
798: nextpg = TAILQ_FIRST(&uobj->memq);
1.53 thorpej 799: PHOLD(curlwp);
1.22 thorpej 800: } else {
801: curoff = start;
1.52 scw 802: nextpg = NULL; /* Quell compiler warning */
1.22 thorpej 803: }
804:
1.46 chs 805: uvm_lock_pageq();
1.22 thorpej 806:
807: /* locked: both page queues and uobj */
1.51 enami 808: for (;;) {
1.22 thorpej 809: if (by_list) {
1.51 enami 810: pg = nextpg;
811: if (pg == &endmp)
812: break;
1.46 chs 813: nextpg = TAILQ_NEXT(pg, listq);
814: if (pg->offset < start || pg->offset >= stop)
1.22 thorpej 815: continue;
816: } else {
1.51 enami 817: if (curoff < stop) {
818: pg = uvm_pagelookup(uobj, curoff);
819: curoff += PAGE_SIZE;
820: } else
821: break;
1.46 chs 822: if (pg == NULL)
1.22 thorpej 823: continue;
824: }
1.46 chs 825: switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
1.41 chs 826:
1.22 thorpej 827: /*
828: * XXX In these first 3 cases, we always just
829: * XXX deactivate the page. We may want to
830: * XXX handle the different cases more specifically
831: * XXX in the future.
832: */
1.46 chs 833:
1.22 thorpej 834: case PGO_CLEANIT|PGO_FREE:
835: case PGO_CLEANIT|PGO_DEACTIVATE:
836: case PGO_DEACTIVATE:
1.25 thorpej 837: deactivate_it:
1.22 thorpej 838: /* skip the page if it's loaned or wired */
1.46 chs 839: if (pg->loan_count != 0 || pg->wire_count != 0)
1.22 thorpej 840: continue;
841:
842: /* ...and deactivate the page. */
1.46 chs 843: pmap_clear_reference(pg);
844: uvm_pagedeactivate(pg);
1.22 thorpej 845: continue;
846:
847: case PGO_FREE:
1.46 chs 848:
1.25 thorpej 849: /*
850: * If there are multiple references to
851: * the object, just deactivate the page.
852: */
1.46 chs 853:
1.25 thorpej 854: if (uobj->uo_refs > 1)
855: goto deactivate_it;
856:
1.22 thorpej 857: /* XXX skip the page if it's loaned or wired */
1.46 chs 858: if (pg->loan_count != 0 || pg->wire_count != 0)
1.22 thorpej 859: continue;
860:
861: /*
1.51 enami 862: * wait and try again if the page is busy.
863: * otherwise free the swap slot and the page.
1.22 thorpej 864: */
1.46 chs 865:
866: pmap_page_protect(pg, VM_PROT_NONE);
1.51 enami 867: if (pg->flags & PG_BUSY) {
868: if (by_list) {
869: TAILQ_INSERT_BEFORE(pg, &curmp, listq);
870: }
1.46 chs 871: pg->flags |= PG_WANTED;
872: uvm_unlock_pageq();
873: UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, 0,
874: "uao_put", 0);
875: simple_lock(&uobj->vmobjlock);
876: uvm_lock_pageq();
1.51 enami 877: if (by_list) {
878: nextpg = TAILQ_NEXT(&curmp, listq);
879: TAILQ_REMOVE(&uobj->memq, &curmp,
880: listq);
881: } else
882: curoff -= PAGE_SIZE;
883: continue;
1.22 thorpej 884: }
1.46 chs 885: uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
886: uvm_pagefree(pg);
1.22 thorpej 887: continue;
888: }
889: }
890: uvm_unlock_pageq();
1.51 enami 891: if (by_list) {
892: TAILQ_REMOVE(&uobj->memq, &endmp, listq);
1.53 thorpej 893: PRELE(curlwp);
1.51 enami 894: }
1.55 pk 895: simple_unlock(&uobj->vmobjlock);
1.46 chs 896: return 0;
1.1 mrg 897: }
898:
899: /*
900: * uao_get: fetch me a page
901: *
902: * we have three cases:
903: * 1: page is resident -> just return the page.
904: * 2: page is zero-fill -> allocate a new page and zero it.
905: * 3: page is swapped out -> fetch the page from swap.
906: *
907: * cases 1 and 2 can be handled with PGO_LOCKED, case 3 cannot.
908: * so, if the "center" page hits case 3 (or any page, with PGO_ALLPAGES),
1.40 chs 909: * then we will need to return EBUSY.
1.1 mrg 910: *
911: * => prefer map unlocked (not required)
912: * => object must be locked! we will _unlock_ it before starting any I/O.
913: * => flags: PGO_ALLPAGES: get all of the pages
914: * PGO_LOCKED: fault data structures are locked
915: * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
916: * => NOTE: caller must check for released pages!!
917: */
1.46 chs 918:
1.5 mrg 919: static int
1.67 thorpej 920: uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
921: int *npagesp, int centeridx, vm_prot_t access_type, int advice, int flags)
1.5 mrg 922: {
923: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.28 kleink 924: voff_t current_offset;
1.52 scw 925: struct vm_page *ptmp = NULL; /* Quell compiler warning */
1.46 chs 926: int lcv, gotpages, maxpages, swslot, error, pageidx;
1.5 mrg 927: boolean_t done;
928: UVMHIST_FUNC("uao_get"); UVMHIST_CALLED(pdhist);
929:
1.27 chs 930: UVMHIST_LOG(pdhist, "aobj=%p offset=%d, flags=%d",
931: aobj, offset, flags,0);
1.37 chs 932:
1.5 mrg 933: /*
934: * get number of pages
935: */
1.46 chs 936:
1.5 mrg 937: maxpages = *npagesp;
938:
939: /*
940: * step 1: handled the case where fault data structures are locked.
941: */
1.1 mrg 942:
1.5 mrg 943: if (flags & PGO_LOCKED) {
1.46 chs 944:
1.5 mrg 945: /*
946: * step 1a: get pages that are already resident. only do
947: * this if the data structures are locked (i.e. the first
948: * time through).
949: */
950:
951: done = TRUE; /* be optimistic */
952: gotpages = 0; /* # of pages we got so far */
953: for (lcv = 0, current_offset = offset ; lcv < maxpages ;
954: lcv++, current_offset += PAGE_SIZE) {
955: /* do we care about this page? if not, skip it */
956: if (pps[lcv] == PGO_DONTCARE)
957: continue;
958: ptmp = uvm_pagelookup(uobj, current_offset);
959:
960: /*
1.30 thorpej 961: * if page is new, attempt to allocate the page,
962: * zero-fill'd.
1.5 mrg 963: */
1.46 chs 964:
965: if (ptmp == NULL && uao_find_swslot(&aobj->u_obj,
1.15 chs 966: current_offset >> PAGE_SHIFT) == 0) {
1.5 mrg 967: ptmp = uvm_pagealloc(uobj, current_offset,
1.30 thorpej 968: NULL, UVM_PGA_ZERO);
1.5 mrg 969: if (ptmp) {
970: /* new page */
1.47 chs 971: ptmp->flags &= ~(PG_FAKE);
1.5 mrg 972: ptmp->pqflags |= PQ_AOBJ;
1.47 chs 973: goto gotpage;
1.5 mrg 974: }
975: }
976:
977: /*
1.46 chs 978: * to be useful must get a non-busy page
1.5 mrg 979: */
1.46 chs 980:
981: if (ptmp == NULL || (ptmp->flags & PG_BUSY) != 0) {
1.5 mrg 982: if (lcv == centeridx ||
983: (flags & PGO_ALLPAGES) != 0)
984: /* need to do a wait or I/O! */
1.41 chs 985: done = FALSE;
1.5 mrg 986: continue;
987: }
988:
989: /*
990: * useful page: busy/lock it and plug it in our
991: * result array
992: */
1.46 chs 993:
1.5 mrg 994: /* caller must un-busy this page */
1.41 chs 995: ptmp->flags |= PG_BUSY;
1.5 mrg 996: UVM_PAGE_OWN(ptmp, "uao_get1");
1.47 chs 997: gotpage:
1.5 mrg 998: pps[lcv] = ptmp;
999: gotpages++;
1.46 chs 1000: }
1.5 mrg 1001:
1002: /*
1003: * step 1b: now we've either done everything needed or we
1004: * to unlock and do some waiting or I/O.
1005: */
1006:
1007: UVMHIST_LOG(pdhist, "<- done (done=%d)", done, 0,0,0);
1008: *npagesp = gotpages;
1009: if (done)
1.46 chs 1010: return 0;
1.5 mrg 1011: else
1.46 chs 1012: return EBUSY;
1.1 mrg 1013: }
1014:
1.5 mrg 1015: /*
1016: * step 2: get non-resident or busy pages.
1017: * object is locked. data structures are unlocked.
1018: */
1019:
1020: for (lcv = 0, current_offset = offset ; lcv < maxpages ;
1021: lcv++, current_offset += PAGE_SIZE) {
1.27 chs 1022:
1.5 mrg 1023: /*
1024: * - skip over pages we've already gotten or don't want
1025: * - skip over pages we don't _have_ to get
1026: */
1.27 chs 1027:
1.5 mrg 1028: if (pps[lcv] != NULL ||
1029: (lcv != centeridx && (flags & PGO_ALLPAGES) == 0))
1030: continue;
1031:
1.27 chs 1032: pageidx = current_offset >> PAGE_SHIFT;
1033:
1.5 mrg 1034: /*
1035: * we have yet to locate the current page (pps[lcv]). we
1036: * first look for a page that is already at the current offset.
1037: * if we find a page, we check to see if it is busy or
1038: * released. if that is the case, then we sleep on the page
1039: * until it is no longer busy or released and repeat the lookup.
1040: * if the page we found is neither busy nor released, then we
1041: * busy it (so we own it) and plug it into pps[lcv]. this
1042: * 'break's the following while loop and indicates we are
1043: * ready to move on to the next page in the "lcv" loop above.
1044: *
1045: * if we exit the while loop with pps[lcv] still set to NULL,
1046: * then it means that we allocated a new busy/fake/clean page
1047: * ptmp in the object and we need to do I/O to fill in the data.
1048: */
1049:
1050: /* top of "pps" while loop */
1051: while (pps[lcv] == NULL) {
1052: /* look for a resident page */
1053: ptmp = uvm_pagelookup(uobj, current_offset);
1054:
1055: /* not resident? allocate one now (if we can) */
1056: if (ptmp == NULL) {
1057:
1058: ptmp = uvm_pagealloc(uobj, current_offset,
1.19 chs 1059: NULL, 0);
1.5 mrg 1060:
1061: /* out of RAM? */
1062: if (ptmp == NULL) {
1063: simple_unlock(&uobj->vmobjlock);
1064: UVMHIST_LOG(pdhist,
1065: "sleeping, ptmp == NULL\n",0,0,0,0);
1066: uvm_wait("uao_getpage");
1067: simple_lock(&uobj->vmobjlock);
1.41 chs 1068: continue;
1.5 mrg 1069: }
1070:
1071: /*
1072: * safe with PQ's unlocked: because we just
1073: * alloc'd the page
1074: */
1.46 chs 1075:
1.5 mrg 1076: ptmp->pqflags |= PQ_AOBJ;
1077:
1.41 chs 1078: /*
1.5 mrg 1079: * got new page ready for I/O. break pps while
1080: * loop. pps[lcv] is still NULL.
1081: */
1.46 chs 1082:
1.5 mrg 1083: break;
1084: }
1085:
1086: /* page is there, see if we need to wait on it */
1.46 chs 1087: if ((ptmp->flags & PG_BUSY) != 0) {
1.5 mrg 1088: ptmp->flags |= PG_WANTED;
1089: UVMHIST_LOG(pdhist,
1090: "sleeping, ptmp->flags 0x%x\n",
1091: ptmp->flags,0,0,0);
1.23 thorpej 1092: UVM_UNLOCK_AND_WAIT(ptmp, &uobj->vmobjlock,
1093: FALSE, "uao_get", 0);
1.5 mrg 1094: simple_lock(&uobj->vmobjlock);
1.46 chs 1095: continue;
1.5 mrg 1096: }
1.41 chs 1097:
1098: /*
1.5 mrg 1099: * if we get here then the page has become resident and
1100: * unbusy between steps 1 and 2. we busy it now (so we
1101: * own it) and set pps[lcv] (so that we exit the while
1102: * loop).
1103: */
1.46 chs 1104:
1.5 mrg 1105: /* we own it, caller must un-busy */
1106: ptmp->flags |= PG_BUSY;
1107: UVM_PAGE_OWN(ptmp, "uao_get2");
1108: pps[lcv] = ptmp;
1109: }
1110:
1111: /*
1112: * if we own the valid page at the correct offset, pps[lcv] will
1113: * point to it. nothing more to do except go to the next page.
1114: */
1.46 chs 1115:
1.5 mrg 1116: if (pps[lcv])
1117: continue; /* next lcv */
1118:
1119: /*
1.41 chs 1120: * we have a "fake/busy/clean" page that we just allocated.
1.5 mrg 1121: * do the needed "i/o", either reading from swap or zeroing.
1122: */
1.46 chs 1123:
1124: swslot = uao_find_swslot(&aobj->u_obj, pageidx);
1.5 mrg 1125:
1126: /*
1127: * just zero the page if there's nothing in swap.
1128: */
1.46 chs 1129:
1130: if (swslot == 0) {
1131:
1.5 mrg 1132: /*
1133: * page hasn't existed before, just zero it.
1134: */
1.46 chs 1135:
1.5 mrg 1136: uvm_pagezero(ptmp);
1.27 chs 1137: } else {
1.5 mrg 1138: UVMHIST_LOG(pdhist, "pagein from swslot %d",
1139: swslot, 0,0,0);
1140:
1141: /*
1142: * page in the swapped-out page.
1143: * unlock object for i/o, relock when done.
1144: */
1.46 chs 1145:
1.5 mrg 1146: simple_unlock(&uobj->vmobjlock);
1.46 chs 1147: error = uvm_swap_get(ptmp, swslot, PGO_SYNCIO);
1.5 mrg 1148: simple_lock(&uobj->vmobjlock);
1149:
1150: /*
1151: * I/O done. check for errors.
1152: */
1.46 chs 1153:
1154: if (error != 0) {
1.5 mrg 1155: UVMHIST_LOG(pdhist, "<- done (error=%d)",
1.46 chs 1156: error,0,0,0);
1.5 mrg 1157: if (ptmp->flags & PG_WANTED)
1.24 thorpej 1158: wakeup(ptmp);
1.27 chs 1159:
1160: /*
1161: * remove the swap slot from the aobj
1162: * and mark the aobj as having no real slot.
1163: * don't free the swap slot, thus preventing
1164: * it from being used again.
1165: */
1.46 chs 1166:
1.27 chs 1167: swslot = uao_set_swslot(&aobj->u_obj, pageidx,
1168: SWSLOT_BAD);
1.57 pk 1169: if (swslot > 0) {
1.45 chs 1170: uvm_swap_markbad(swslot, 1);
1171: }
1.27 chs 1172:
1.5 mrg 1173: uvm_lock_pageq();
1174: uvm_pagefree(ptmp);
1175: uvm_unlock_pageq();
1176: simple_unlock(&uobj->vmobjlock);
1.46 chs 1177: return error;
1.5 mrg 1178: }
1179: }
1180:
1.41 chs 1181: /*
1.5 mrg 1182: * we got the page! clear the fake flag (indicates valid
1183: * data now in page) and plug into our result array. note
1.41 chs 1184: * that page is still busy.
1.5 mrg 1185: *
1186: * it is the callers job to:
1187: * => check if the page is released
1188: * => unbusy the page
1189: * => activate the page
1190: */
1191:
1.46 chs 1192: ptmp->flags &= ~PG_FAKE;
1.5 mrg 1193: pps[lcv] = ptmp;
1.46 chs 1194: }
1.1 mrg 1195:
1196: /*
1.5 mrg 1197: * finally, unlock object and return.
1198: */
1.1 mrg 1199:
1200: simple_unlock(&uobj->vmobjlock);
1.5 mrg 1201: UVMHIST_LOG(pdhist, "<- done (OK)",0,0,0,0);
1.46 chs 1202: return 0;
1.1 mrg 1203: }
1204:
1205: /*
1.18 chs 1206: * uao_dropswap: release any swap resources from this aobj page.
1.41 chs 1207: *
1.18 chs 1208: * => aobj must be locked or have a reference count of 0.
1209: */
1210:
1211: void
1.67 thorpej 1212: uao_dropswap(struct uvm_object *uobj, int pageidx)
1.18 chs 1213: {
1214: int slot;
1215:
1216: slot = uao_set_swslot(uobj, pageidx, 0);
1217: if (slot) {
1218: uvm_swap_free(slot, 1);
1219: }
1.27 chs 1220: }
1221:
1222: /*
1223: * page in every page in every aobj that is paged-out to a range of swslots.
1.41 chs 1224: *
1.27 chs 1225: * => nothing should be locked.
1226: * => returns TRUE if pagein was aborted due to lack of memory.
1227: */
1.46 chs 1228:
1.27 chs 1229: boolean_t
1.67 thorpej 1230: uao_swap_off(int startslot, int endslot)
1.27 chs 1231: {
1232: struct uvm_aobj *aobj, *nextaobj;
1.46 chs 1233: boolean_t rv;
1.27 chs 1234:
1235: /*
1236: * walk the list of all aobjs.
1237: */
1238:
1239: restart:
1240: simple_lock(&uao_list_lock);
1241: for (aobj = LIST_FIRST(&uao_list);
1242: aobj != NULL;
1243: aobj = nextaobj) {
1244:
1245: /*
1.46 chs 1246: * try to get the object lock, start all over if we fail.
1.27 chs 1247: * most of the time we'll get the aobj lock,
1248: * so this should be a rare case.
1249: */
1.46 chs 1250:
1.27 chs 1251: if (!simple_lock_try(&aobj->u_obj.vmobjlock)) {
1252: simple_unlock(&uao_list_lock);
1253: goto restart;
1254: }
1255:
1256: /*
1257: * add a ref to the aobj so it doesn't disappear
1258: * while we're working.
1259: */
1.46 chs 1260:
1.27 chs 1261: uao_reference_locked(&aobj->u_obj);
1262:
1263: /*
1264: * now it's safe to unlock the uao list.
1265: */
1.46 chs 1266:
1.27 chs 1267: simple_unlock(&uao_list_lock);
1268:
1269: /*
1270: * page in any pages in the swslot range.
1271: * if there's an error, abort and return the error.
1272: */
1.46 chs 1273:
1.27 chs 1274: rv = uao_pagein(aobj, startslot, endslot);
1275: if (rv) {
1276: uao_detach_locked(&aobj->u_obj);
1277: return rv;
1278: }
1279:
1280: /*
1281: * we're done with this aobj.
1282: * relock the list and drop our ref on the aobj.
1283: */
1.46 chs 1284:
1.27 chs 1285: simple_lock(&uao_list_lock);
1286: nextaobj = LIST_NEXT(aobj, u_list);
1287: uao_detach_locked(&aobj->u_obj);
1288: }
1289:
1290: /*
1291: * done with traversal, unlock the list
1292: */
1293: simple_unlock(&uao_list_lock);
1294: return FALSE;
1295: }
1296:
1297:
1298: /*
1299: * page in any pages from aobj in the given range.
1300: *
1301: * => aobj must be locked and is returned locked.
1302: * => returns TRUE if pagein was aborted due to lack of memory.
1303: */
1304: static boolean_t
1.67 thorpej 1305: uao_pagein(struct uvm_aobj *aobj, int startslot, int endslot)
1.27 chs 1306: {
1307: boolean_t rv;
1308:
1309: if (UAO_USES_SWHASH(aobj)) {
1310: struct uao_swhash_elt *elt;
1.65 christos 1311: int buck;
1.27 chs 1312:
1313: restart:
1.65 christos 1314: for (buck = aobj->u_swhashmask; buck >= 0; buck--) {
1315: for (elt = LIST_FIRST(&aobj->u_swhash[buck]);
1.27 chs 1316: elt != NULL;
1317: elt = LIST_NEXT(elt, list)) {
1318: int i;
1319:
1320: for (i = 0; i < UAO_SWHASH_CLUSTER_SIZE; i++) {
1321: int slot = elt->slots[i];
1322:
1323: /*
1324: * if the slot isn't in range, skip it.
1325: */
1.46 chs 1326:
1.41 chs 1327: if (slot < startslot ||
1.27 chs 1328: slot >= endslot) {
1329: continue;
1330: }
1331:
1332: /*
1333: * process the page,
1334: * the start over on this object
1335: * since the swhash elt
1336: * may have been freed.
1337: */
1.46 chs 1338:
1.27 chs 1339: rv = uao_pagein_page(aobj,
1340: UAO_SWHASH_ELT_PAGEIDX_BASE(elt) + i);
1341: if (rv) {
1342: return rv;
1343: }
1344: goto restart;
1345: }
1346: }
1347: }
1348: } else {
1349: int i;
1350:
1351: for (i = 0; i < aobj->u_pages; i++) {
1352: int slot = aobj->u_swslots[i];
1353:
1354: /*
1355: * if the slot isn't in range, skip it
1356: */
1.46 chs 1357:
1.27 chs 1358: if (slot < startslot || slot >= endslot) {
1359: continue;
1360: }
1361:
1362: /*
1363: * process the page.
1364: */
1.46 chs 1365:
1.27 chs 1366: rv = uao_pagein_page(aobj, i);
1367: if (rv) {
1368: return rv;
1369: }
1370: }
1371: }
1372:
1373: return FALSE;
1374: }
1375:
1376: /*
1377: * page in a page from an aobj. used for swap_off.
1378: * returns TRUE if pagein was aborted due to lack of memory.
1379: *
1380: * => aobj must be locked and is returned locked.
1381: */
1.46 chs 1382:
1.27 chs 1383: static boolean_t
1.67 thorpej 1384: uao_pagein_page(struct uvm_aobj *aobj, int pageidx)
1.27 chs 1385: {
1386: struct vm_page *pg;
1.57 pk 1387: int rv, npages;
1.27 chs 1388:
1389: pg = NULL;
1390: npages = 1;
1391: /* locked: aobj */
1392: rv = uao_get(&aobj->u_obj, pageidx << PAGE_SHIFT,
1393: &pg, &npages, 0, VM_PROT_READ|VM_PROT_WRITE, 0, 0);
1394: /* unlocked: aobj */
1395:
1396: /*
1397: * relock and finish up.
1398: */
1.46 chs 1399:
1.27 chs 1400: simple_lock(&aobj->u_obj.vmobjlock);
1401: switch (rv) {
1.40 chs 1402: case 0:
1.27 chs 1403: break;
1404:
1.40 chs 1405: case EIO:
1406: case ERESTART:
1.46 chs 1407:
1.27 chs 1408: /*
1409: * nothing more to do on errors.
1.40 chs 1410: * ERESTART can only mean that the anon was freed,
1.27 chs 1411: * so again there's nothing to do.
1412: */
1.46 chs 1413:
1.27 chs 1414: return FALSE;
1.59 pk 1415:
1416: default:
1417: return TRUE;
1.27 chs 1418: }
1419:
1420: /*
1421: * ok, we've got the page now.
1422: * mark it as dirty, clear its swslot and un-busy it.
1423: */
1.57 pk 1424: uao_dropswap(&aobj->u_obj, pageidx);
1.27 chs 1425:
1426: /*
1.46 chs 1427: * deactivate the page (to make sure it's on a page queue).
1.27 chs 1428: */
1429: uvm_lock_pageq();
1.58 pk 1430: if (pg->wire_count == 0)
1431: uvm_pagedeactivate(pg);
1.27 chs 1432: uvm_unlock_pageq();
1.56 yamt 1433:
1.59 pk 1434: if (pg->flags & PG_WANTED) {
1435: wakeup(pg);
1436: }
1437: pg->flags &= ~(PG_WANTED|PG_BUSY|PG_CLEAN|PG_FAKE);
1.56 yamt 1438: UVM_PAGE_OWN(pg, NULL);
1439:
1.27 chs 1440: return FALSE;
1.1 mrg 1441: }
CVSweb <webmaster@jp.NetBSD.org>