[BACK]Return to uvm_aobj.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / uvm

Annotation of src/sys/uvm/uvm_aobj.c, Revision 1.48

1.48    ! chs         1: /*     $NetBSD: uvm_aobj.c,v 1.47 2001/11/06 08:07:49 chs Exp $        */
1.6       mrg         2:
1.7       chs         3: /*
                      4:  * Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
                      5:  *                    Washington University.
                      6:  * All rights reserved.
                      7:  *
                      8:  * Redistribution and use in source and binary forms, with or without
                      9:  * modification, are permitted provided that the following conditions
                     10:  * are met:
                     11:  * 1. Redistributions of source code must retain the above copyright
                     12:  *    notice, this list of conditions and the following disclaimer.
                     13:  * 2. Redistributions in binary form must reproduce the above copyright
                     14:  *    notice, this list of conditions and the following disclaimer in the
                     15:  *    documentation and/or other materials provided with the distribution.
                     16:  * 3. All advertising materials mentioning features or use of this software
                     17:  *    must display the following acknowledgement:
                     18:  *      This product includes software developed by Charles D. Cranor and
                     19:  *      Washington University.
                     20:  * 4. The name of the author may not be used to endorse or promote products
                     21:  *    derived from this software without specific prior written permission.
                     22:  *
                     23:  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
                     24:  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
                     25:  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
                     26:  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
                     27:  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
                     28:  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
                     29:  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
                     30:  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
                     31:  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
                     32:  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
                     33:  *
1.4       mrg        34:  * from: Id: uvm_aobj.c,v 1.1.2.5 1998/02/06 05:14:38 chs Exp
                     35:  */
1.7       chs        36: /*
                     37:  * uvm_aobj.c: anonymous memory uvm_object pager
                     38:  *
                     39:  * author: Chuck Silvers <chuq@chuq.com>
                     40:  * started: Jan-1998
                     41:  *
                     42:  * - design mostly from Chuck Cranor
                     43:  */
                     44:
                     45: #include "opt_uvmhist.h"
1.1       mrg        46:
                     47: #include <sys/param.h>
                     48: #include <sys/systm.h>
                     49: #include <sys/proc.h>
                     50: #include <sys/malloc.h>
1.37      chs        51: #include <sys/kernel.h>
1.12      thorpej    52: #include <sys/pool.h>
1.27      chs        53: #include <sys/kernel.h>
1.1       mrg        54:
                     55: #include <uvm/uvm.h>
                     56:
                     57: /*
                     58:  * an aobj manages anonymous-memory backed uvm_objects.   in addition
                     59:  * to keeping the list of resident pages, it also keeps a list of
                     60:  * allocated swap blocks.  depending on the size of the aobj this list
                     61:  * of allocated swap blocks is either stored in an array (small objects)
                     62:  * or in a hash table (large objects).
                     63:  */
                     64:
                     65: /*
                     66:  * local structures
                     67:  */
                     68:
                     69: /*
                     70:  * for hash tables, we break the address space of the aobj into blocks
                     71:  * of UAO_SWHASH_CLUSTER_SIZE pages.   we require the cluster size to
                     72:  * be a power of two.
                     73:  */
                     74:
                     75: #define UAO_SWHASH_CLUSTER_SHIFT 4
                     76: #define UAO_SWHASH_CLUSTER_SIZE (1 << UAO_SWHASH_CLUSTER_SHIFT)
                     77:
                     78: /* get the "tag" for this page index */
                     79: #define UAO_SWHASH_ELT_TAG(PAGEIDX) \
                     80:        ((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT)
                     81:
                     82: /* given an ELT and a page index, find the swap slot */
                     83: #define UAO_SWHASH_ELT_PAGESLOT(ELT, PAGEIDX) \
                     84:        ((ELT)->slots[(PAGEIDX) & (UAO_SWHASH_CLUSTER_SIZE - 1)])
                     85:
                     86: /* given an ELT, return its pageidx base */
                     87: #define UAO_SWHASH_ELT_PAGEIDX_BASE(ELT) \
                     88:        ((ELT)->tag << UAO_SWHASH_CLUSTER_SHIFT)
                     89:
                     90: /*
                     91:  * the swhash hash function
                     92:  */
1.46      chs        93:
1.1       mrg        94: #define UAO_SWHASH_HASH(AOBJ, PAGEIDX) \
                     95:        (&(AOBJ)->u_swhash[(((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT) \
                     96:                            & (AOBJ)->u_swhashmask)])
                     97:
                     98: /*
                     99:  * the swhash threshhold determines if we will use an array or a
                    100:  * hash table to store the list of allocated swap blocks.
                    101:  */
                    102:
                    103: #define UAO_SWHASH_THRESHOLD (UAO_SWHASH_CLUSTER_SIZE * 4)
                    104: #define UAO_USES_SWHASH(AOBJ) \
                    105:        ((AOBJ)->u_pages > UAO_SWHASH_THRESHOLD)        /* use hash? */
                    106:
                    107: /*
1.3       chs       108:  * the number of buckets in a swhash, with an upper bound
1.1       mrg       109:  */
1.46      chs       110:
1.1       mrg       111: #define UAO_SWHASH_MAXBUCKETS 256
                    112: #define UAO_SWHASH_BUCKETS(AOBJ) \
1.46      chs       113:        (MIN((AOBJ)->u_pages >> UAO_SWHASH_CLUSTER_SHIFT, \
1.1       mrg       114:             UAO_SWHASH_MAXBUCKETS))
                    115:
                    116:
                    117: /*
                    118:  * uao_swhash_elt: when a hash table is being used, this structure defines
                    119:  * the format of an entry in the bucket list.
                    120:  */
                    121:
                    122: struct uao_swhash_elt {
1.5       mrg       123:        LIST_ENTRY(uao_swhash_elt) list;        /* the hash list */
1.28      kleink    124:        voff_t tag;                             /* our 'tag' */
1.5       mrg       125:        int count;                              /* our number of active slots */
                    126:        int slots[UAO_SWHASH_CLUSTER_SIZE];     /* the slots */
1.1       mrg       127: };
                    128:
                    129: /*
                    130:  * uao_swhash: the swap hash table structure
                    131:  */
                    132:
                    133: LIST_HEAD(uao_swhash, uao_swhash_elt);
                    134:
1.12      thorpej   135: /*
                    136:  * uao_swhash_elt_pool: pool of uao_swhash_elt structures
                    137:  */
                    138:
                    139: struct pool uao_swhash_elt_pool;
1.1       mrg       140:
                    141: /*
                    142:  * uvm_aobj: the actual anon-backed uvm_object
                    143:  *
                    144:  * => the uvm_object is at the top of the structure, this allows
1.46      chs       145:  *   (struct uvm_aobj *) == (struct uvm_object *)
1.1       mrg       146:  * => only one of u_swslots and u_swhash is used in any given aobj
                    147:  */
                    148:
                    149: struct uvm_aobj {
1.5       mrg       150:        struct uvm_object u_obj; /* has: lock, pgops, memq, #pages, #refs */
1.11      drochner  151:        int u_pages;             /* number of pages in entire object */
1.5       mrg       152:        int u_flags;             /* the flags (see uvm_aobj.h) */
                    153:        int *u_swslots;          /* array of offset->swapslot mappings */
                    154:                                 /*
                    155:                                  * hashtable of offset->swapslot mappings
                    156:                                  * (u_swhash is an array of bucket heads)
                    157:                                  */
                    158:        struct uao_swhash *u_swhash;
                    159:        u_long u_swhashmask;            /* mask for hashtable */
                    160:        LIST_ENTRY(uvm_aobj) u_list;    /* global list of aobjs */
1.1       mrg       161: };
                    162:
                    163: /*
1.12      thorpej   164:  * uvm_aobj_pool: pool of uvm_aobj structures
                    165:  */
                    166:
                    167: struct pool uvm_aobj_pool;
                    168:
                    169: /*
1.1       mrg       170:  * local functions
                    171:  */
                    172:
1.46      chs       173: static struct uao_swhash_elt *uao_find_swhash_elt
                    174:     __P((struct uvm_aobj *, int, boolean_t));
                    175:
                    176: static void    uao_free __P((struct uvm_aobj *));
                    177: static int     uao_get __P((struct uvm_object *, voff_t, struct vm_page **,
                    178:                    int *, int, vm_prot_t, int, int));
                    179: static boolean_t uao_put __P((struct uvm_object *, voff_t, voff_t, int));
                    180: static boolean_t uao_pagein __P((struct uvm_aobj *, int, int));
                    181: static boolean_t uao_pagein_page __P((struct uvm_aobj *, int));
1.1       mrg       182:
                    183: /*
                    184:  * aobj_pager
1.41      chs       185:  *
1.1       mrg       186:  * note that some functions (e.g. put) are handled elsewhere
                    187:  */
                    188:
                    189: struct uvm_pagerops aobj_pager = {
1.27      chs       190:        NULL,                   /* init */
1.5       mrg       191:        uao_reference,          /* reference */
                    192:        uao_detach,             /* detach */
                    193:        NULL,                   /* fault */
                    194:        uao_get,                /* get */
1.46      chs       195:        uao_put,                /* flush */
1.1       mrg       196: };
                    197:
                    198: /*
                    199:  * uao_list: global list of active aobjs, locked by uao_list_lock
                    200:  */
                    201:
                    202: static LIST_HEAD(aobjlist, uvm_aobj) uao_list;
1.42      chs       203: static struct simplelock uao_list_lock;
1.1       mrg       204:
                    205: /*
                    206:  * functions
                    207:  */
                    208:
                    209: /*
                    210:  * hash table/array related functions
                    211:  */
                    212:
                    213: /*
                    214:  * uao_find_swhash_elt: find (or create) a hash table entry for a page
                    215:  * offset.
                    216:  *
                    217:  * => the object should be locked by the caller
                    218:  */
                    219:
1.5       mrg       220: static struct uao_swhash_elt *
                    221: uao_find_swhash_elt(aobj, pageidx, create)
                    222:        struct uvm_aobj *aobj;
                    223:        int pageidx;
                    224:        boolean_t create;
                    225: {
                    226:        struct uao_swhash *swhash;
                    227:        struct uao_swhash_elt *elt;
1.28      kleink    228:        voff_t page_tag;
1.1       mrg       229:
1.45      chs       230:        swhash = UAO_SWHASH_HASH(aobj, pageidx);
                    231:        page_tag = UAO_SWHASH_ELT_TAG(pageidx);
1.1       mrg       232:
1.5       mrg       233:        /*
                    234:         * now search the bucket for the requested tag
                    235:         */
1.45      chs       236:
1.37      chs       237:        LIST_FOREACH(elt, swhash, list) {
1.45      chs       238:                if (elt->tag == page_tag) {
                    239:                        return elt;
                    240:                }
1.5       mrg       241:        }
1.45      chs       242:        if (!create) {
1.5       mrg       243:                return NULL;
1.45      chs       244:        }
1.5       mrg       245:
                    246:        /*
1.12      thorpej   247:         * allocate a new entry for the bucket and init/insert it in
1.5       mrg       248:         */
1.45      chs       249:
                    250:        elt = pool_get(&uao_swhash_elt_pool, PR_NOWAIT);
                    251:        if (elt == NULL) {
                    252:                return NULL;
                    253:        }
1.5       mrg       254:        LIST_INSERT_HEAD(swhash, elt, list);
                    255:        elt->tag = page_tag;
                    256:        elt->count = 0;
1.9       perry     257:        memset(elt->slots, 0, sizeof(elt->slots));
1.45      chs       258:        return elt;
1.1       mrg       259: }
                    260:
                    261: /*
                    262:  * uao_find_swslot: find the swap slot number for an aobj/pageidx
                    263:  *
1.41      chs       264:  * => object must be locked by caller
1.1       mrg       265:  */
1.46      chs       266:
                    267: int
                    268: uao_find_swslot(uobj, pageidx)
                    269:        struct uvm_object *uobj;
1.11      drochner  270:        int pageidx;
1.1       mrg       271: {
1.46      chs       272:        struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
                    273:        struct uao_swhash_elt *elt;
1.1       mrg       274:
1.5       mrg       275:        /*
                    276:         * if noswap flag is set, then we never return a slot
                    277:         */
1.1       mrg       278:
1.5       mrg       279:        if (aobj->u_flags & UAO_FLAG_NOSWAP)
                    280:                return(0);
1.1       mrg       281:
1.5       mrg       282:        /*
                    283:         * if hashing, look in hash table.
                    284:         */
1.1       mrg       285:
1.5       mrg       286:        if (UAO_USES_SWHASH(aobj)) {
1.46      chs       287:                elt = uao_find_swhash_elt(aobj, pageidx, FALSE);
1.5       mrg       288:                if (elt)
                    289:                        return(UAO_SWHASH_ELT_PAGESLOT(elt, pageidx));
                    290:                else
1.31      thorpej   291:                        return(0);
1.5       mrg       292:        }
1.1       mrg       293:
1.41      chs       294:        /*
1.5       mrg       295:         * otherwise, look in the array
                    296:         */
1.46      chs       297:
1.5       mrg       298:        return(aobj->u_swslots[pageidx]);
1.1       mrg       299: }
                    300:
                    301: /*
                    302:  * uao_set_swslot: set the swap slot for a page in an aobj.
                    303:  *
                    304:  * => setting a slot to zero frees the slot
                    305:  * => object must be locked by caller
1.45      chs       306:  * => we return the old slot number, or -1 if we failed to allocate
                    307:  *    memory to record the new slot number
1.1       mrg       308:  */
1.46      chs       309:
1.5       mrg       310: int
                    311: uao_set_swslot(uobj, pageidx, slot)
                    312:        struct uvm_object *uobj;
                    313:        int pageidx, slot;
                    314: {
                    315:        struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.45      chs       316:        struct uao_swhash_elt *elt;
1.5       mrg       317:        int oldslot;
                    318:        UVMHIST_FUNC("uao_set_swslot"); UVMHIST_CALLED(pdhist);
                    319:        UVMHIST_LOG(pdhist, "aobj %p pageidx %d slot %d",
                    320:            aobj, pageidx, slot, 0);
1.1       mrg       321:
1.5       mrg       322:        /*
1.46      chs       323:         * if noswap flag is set, then we can't set a non-zero slot.
1.5       mrg       324:         */
1.1       mrg       325:
1.5       mrg       326:        if (aobj->u_flags & UAO_FLAG_NOSWAP) {
                    327:                if (slot == 0)
1.46      chs       328:                        return(0);
1.1       mrg       329:
1.5       mrg       330:                printf("uao_set_swslot: uobj = %p\n", uobj);
1.46      chs       331:                panic("uao_set_swslot: NOSWAP object");
1.5       mrg       332:        }
1.1       mrg       333:
1.5       mrg       334:        /*
                    335:         * are we using a hash table?  if so, add it in the hash.
                    336:         */
1.1       mrg       337:
1.5       mrg       338:        if (UAO_USES_SWHASH(aobj)) {
1.39      chs       339:
1.12      thorpej   340:                /*
                    341:                 * Avoid allocating an entry just to free it again if
                    342:                 * the page had not swap slot in the first place, and
                    343:                 * we are freeing.
                    344:                 */
1.39      chs       345:
1.46      chs       346:                elt = uao_find_swhash_elt(aobj, pageidx, slot != 0);
1.12      thorpej   347:                if (elt == NULL) {
1.45      chs       348:                        return slot ? -1 : 0;
1.12      thorpej   349:                }
1.5       mrg       350:
                    351:                oldslot = UAO_SWHASH_ELT_PAGESLOT(elt, pageidx);
                    352:                UAO_SWHASH_ELT_PAGESLOT(elt, pageidx) = slot;
                    353:
                    354:                /*
                    355:                 * now adjust the elt's reference counter and free it if we've
                    356:                 * dropped it to zero.
                    357:                 */
                    358:
                    359:                if (slot) {
                    360:                        if (oldslot == 0)
                    361:                                elt->count++;
1.45      chs       362:                } else {
                    363:                        if (oldslot)
1.5       mrg       364:                                elt->count--;
                    365:
                    366:                        if (elt->count == 0) {
                    367:                                LIST_REMOVE(elt, list);
1.12      thorpej   368:                                pool_put(&uao_swhash_elt_pool, elt);
1.5       mrg       369:                        }
                    370:                }
1.41      chs       371:        } else {
1.5       mrg       372:                /* we are using an array */
                    373:                oldslot = aobj->u_swslots[pageidx];
                    374:                aobj->u_swslots[pageidx] = slot;
                    375:        }
                    376:        return (oldslot);
1.1       mrg       377: }
                    378:
                    379: /*
                    380:  * end of hash/array functions
                    381:  */
                    382:
                    383: /*
                    384:  * uao_free: free all resources held by an aobj, and then free the aobj
                    385:  *
                    386:  * => the aobj should be dead
                    387:  */
1.46      chs       388:
1.1       mrg       389: static void
                    390: uao_free(aobj)
1.5       mrg       391:        struct uvm_aobj *aobj;
1.1       mrg       392: {
1.46      chs       393:        int swpgonlydelta = 0;
1.1       mrg       394:
1.27      chs       395:        simple_unlock(&aobj->u_obj.vmobjlock);
1.5       mrg       396:        if (UAO_USES_SWHASH(aobj)) {
                    397:                int i, hashbuckets = aobj->u_swhashmask + 1;
1.1       mrg       398:
1.5       mrg       399:                /*
                    400:                 * free the swslots from each hash bucket,
                    401:                 * then the hash bucket, and finally the hash table itself.
                    402:                 */
1.46      chs       403:
1.5       mrg       404:                for (i = 0; i < hashbuckets; i++) {
                    405:                        struct uao_swhash_elt *elt, *next;
                    406:
1.27      chs       407:                        for (elt = LIST_FIRST(&aobj->u_swhash[i]);
                    408:                             elt != NULL;
                    409:                             elt = next) {
1.5       mrg       410:                                int j;
                    411:
1.27      chs       412:                                for (j = 0; j < UAO_SWHASH_CLUSTER_SIZE; j++) {
1.5       mrg       413:                                        int slot = elt->slots[j];
                    414:
1.37      chs       415:                                        if (slot == 0) {
                    416:                                                continue;
                    417:                                        }
                    418:                                        uvm_swap_free(slot, 1);
1.46      chs       419:                                        swpgonlydelta++;
1.5       mrg       420:                                }
                    421:
1.27      chs       422:                                next = LIST_NEXT(elt, list);
1.12      thorpej   423:                                pool_put(&uao_swhash_elt_pool, elt);
1.5       mrg       424:                        }
                    425:                }
1.34      thorpej   426:                free(aobj->u_swhash, M_UVMAOBJ);
1.5       mrg       427:        } else {
                    428:                int i;
                    429:
                    430:                /*
                    431:                 * free the array
                    432:                 */
                    433:
1.27      chs       434:                for (i = 0; i < aobj->u_pages; i++) {
1.5       mrg       435:                        int slot = aobj->u_swslots[i];
                    436:
1.18      chs       437:                        if (slot) {
1.5       mrg       438:                                uvm_swap_free(slot, 1);
1.46      chs       439:                                swpgonlydelta++;
1.18      chs       440:                        }
1.5       mrg       441:                }
1.34      thorpej   442:                free(aobj->u_swslots, M_UVMAOBJ);
1.1       mrg       443:        }
                    444:
1.5       mrg       445:        /*
                    446:         * finally free the aobj itself
                    447:         */
1.46      chs       448:
1.12      thorpej   449:        pool_put(&uvm_aobj_pool, aobj);
1.46      chs       450:
                    451:        /*
                    452:         * adjust the counter of pages only in swap for all
                    453:         * the swap slots we've freed.
                    454:         */
                    455:
1.48    ! chs       456:        if (swpgonlydelta > 0) {
        !           457:                simple_lock(&uvm.swap_data_lock);
        !           458:                KASSERT(uvmexp.swpgonly >= swpgonlydelta);
        !           459:                uvmexp.swpgonly -= swpgonlydelta;
        !           460:                simple_unlock(&uvm.swap_data_lock);
        !           461:        }
1.1       mrg       462: }
                    463:
                    464: /*
                    465:  * pager functions
                    466:  */
                    467:
                    468: /*
                    469:  * uao_create: create an aobj of the given size and return its uvm_object.
                    470:  *
                    471:  * => for normal use, flags are always zero
                    472:  * => for the kernel object, the flags are:
                    473:  *     UAO_FLAG_KERNOBJ - allocate the kernel object (can only happen once)
                    474:  *     UAO_FLAG_KERNSWAP - enable swapping of kernel object ("           ")
                    475:  */
1.46      chs       476:
1.5       mrg       477: struct uvm_object *
                    478: uao_create(size, flags)
1.10      eeh       479:        vsize_t size;
1.5       mrg       480:        int flags;
                    481: {
1.46      chs       482:        static struct uvm_aobj kernel_object_store;
                    483:        static int kobj_alloced = 0;
1.15      chs       484:        int pages = round_page(size) >> PAGE_SHIFT;
1.5       mrg       485:        struct uvm_aobj *aobj;
1.1       mrg       486:
1.5       mrg       487:        /*
1.27      chs       488:         * malloc a new aobj unless we are asked for the kernel object
                    489:         */
1.5       mrg       490:
1.46      chs       491:        if (flags & UAO_FLAG_KERNOBJ) {
                    492:                KASSERT(!kobj_alloced);
1.5       mrg       493:                aobj = &kernel_object_store;
                    494:                aobj->u_pages = pages;
1.46      chs       495:                aobj->u_flags = UAO_FLAG_NOSWAP;
1.5       mrg       496:                aobj->u_obj.uo_refs = UVM_OBJ_KERN;
                    497:                kobj_alloced = UAO_FLAG_KERNOBJ;
                    498:        } else if (flags & UAO_FLAG_KERNSWAP) {
1.46      chs       499:                KASSERT(kobj_alloced == UAO_FLAG_KERNOBJ);
1.5       mrg       500:                aobj = &kernel_object_store;
                    501:                kobj_alloced = UAO_FLAG_KERNSWAP;
1.46      chs       502:        } else {
1.12      thorpej   503:                aobj = pool_get(&uvm_aobj_pool, PR_WAITOK);
1.5       mrg       504:                aobj->u_pages = pages;
1.46      chs       505:                aobj->u_flags = 0;
                    506:                aobj->u_obj.uo_refs = 1;
1.5       mrg       507:        }
1.1       mrg       508:
1.5       mrg       509:        /*
                    510:         * allocate hash/array if necessary
                    511:         *
                    512:         * note: in the KERNSWAP case no need to worry about locking since
                    513:         * we are still booting we should be the only thread around.
                    514:         */
1.46      chs       515:
1.5       mrg       516:        if (flags == 0 || (flags & UAO_FLAG_KERNSWAP) != 0) {
                    517:                int mflags = (flags & UAO_FLAG_KERNSWAP) != 0 ?
                    518:                    M_NOWAIT : M_WAITOK;
                    519:
                    520:                /* allocate hash table or array depending on object size */
1.27      chs       521:                if (UAO_USES_SWHASH(aobj)) {
1.5       mrg       522:                        aobj->u_swhash = hashinit(UAO_SWHASH_BUCKETS(aobj),
1.35      ad        523:                            HASH_LIST, M_UVMAOBJ, mflags, &aobj->u_swhashmask);
1.5       mrg       524:                        if (aobj->u_swhash == NULL)
                    525:                                panic("uao_create: hashinit swhash failed");
                    526:                } else {
1.34      thorpej   527:                        aobj->u_swslots = malloc(pages * sizeof(int),
1.5       mrg       528:                            M_UVMAOBJ, mflags);
                    529:                        if (aobj->u_swslots == NULL)
                    530:                                panic("uao_create: malloc swslots failed");
1.9       perry     531:                        memset(aobj->u_swslots, 0, pages * sizeof(int));
1.5       mrg       532:                }
                    533:
                    534:                if (flags) {
                    535:                        aobj->u_flags &= ~UAO_FLAG_NOSWAP; /* clear noswap */
                    536:                        return(&aobj->u_obj);
                    537:                }
                    538:        }
                    539:
                    540:        /*
                    541:         * init aobj fields
                    542:         */
1.46      chs       543:
1.5       mrg       544:        simple_lock_init(&aobj->u_obj.vmobjlock);
                    545:        aobj->u_obj.pgops = &aobj_pager;
                    546:        TAILQ_INIT(&aobj->u_obj.memq);
                    547:        aobj->u_obj.uo_npages = 0;
1.1       mrg       548:
1.5       mrg       549:        /*
                    550:         * now that aobj is ready, add it to the global list
                    551:         */
1.46      chs       552:
1.5       mrg       553:        simple_lock(&uao_list_lock);
                    554:        LIST_INSERT_HEAD(&uao_list, aobj, u_list);
                    555:        simple_unlock(&uao_list_lock);
                    556:        return(&aobj->u_obj);
1.1       mrg       557: }
                    558:
                    559:
                    560:
                    561: /*
                    562:  * uao_init: set up aobj pager subsystem
                    563:  *
                    564:  * => called at boot time from uvm_pager_init()
                    565:  */
1.46      chs       566:
1.27      chs       567: void
1.46      chs       568: uao_init(void)
1.5       mrg       569: {
1.12      thorpej   570:        static int uao_initialized;
                    571:
                    572:        if (uao_initialized)
                    573:                return;
                    574:        uao_initialized = TRUE;
1.5       mrg       575:        LIST_INIT(&uao_list);
                    576:        simple_lock_init(&uao_list_lock);
1.12      thorpej   577:
1.14      thorpej   578:        /*
                    579:         * NOTE: Pages fror this pool must not come from a pageable
                    580:         * kernel map!
                    581:         */
1.46      chs       582:
1.12      thorpej   583:        pool_init(&uao_swhash_elt_pool, sizeof(struct uao_swhash_elt),
1.13      thorpej   584:            0, 0, 0, "uaoeltpl", 0, NULL, NULL, M_UVMAOBJ);
1.12      thorpej   585:        pool_init(&uvm_aobj_pool, sizeof(struct uvm_aobj), 0, 0, 0,
                    586:            "aobjpl", 0,
                    587:            pool_page_alloc_nointr, pool_page_free_nointr, M_UVMAOBJ);
1.1       mrg       588: }
                    589:
                    590: /*
                    591:  * uao_reference: add a ref to an aobj
                    592:  *
1.27      chs       593:  * => aobj must be unlocked
                    594:  * => just lock it and call the locked version
1.1       mrg       595:  */
1.46      chs       596:
1.5       mrg       597: void
                    598: uao_reference(uobj)
                    599:        struct uvm_object *uobj;
1.1       mrg       600: {
1.27      chs       601:        simple_lock(&uobj->vmobjlock);
                    602:        uao_reference_locked(uobj);
                    603:        simple_unlock(&uobj->vmobjlock);
                    604: }
                    605:
                    606: /*
                    607:  * uao_reference_locked: add a ref to an aobj that is already locked
                    608:  *
                    609:  * => aobj must be locked
                    610:  * this needs to be separate from the normal routine
                    611:  * since sometimes we need to add a reference to an aobj when
                    612:  * it's already locked.
                    613:  */
1.46      chs       614:
1.27      chs       615: void
                    616: uao_reference_locked(uobj)
                    617:        struct uvm_object *uobj;
                    618: {
1.5       mrg       619:        UVMHIST_FUNC("uao_reference"); UVMHIST_CALLED(maphist);
1.1       mrg       620:
1.5       mrg       621:        /*
                    622:         * kernel_object already has plenty of references, leave it alone.
                    623:         */
1.1       mrg       624:
1.20      thorpej   625:        if (UVM_OBJ_IS_KERN_OBJECT(uobj))
1.5       mrg       626:                return;
1.1       mrg       627:
1.46      chs       628:        uobj->uo_refs++;
1.41      chs       629:        UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)",
1.27      chs       630:                    uobj, uobj->uo_refs,0,0);
1.1       mrg       631: }
                    632:
                    633: /*
                    634:  * uao_detach: drop a reference to an aobj
                    635:  *
1.27      chs       636:  * => aobj must be unlocked
                    637:  * => just lock it and call the locked version
1.1       mrg       638:  */
1.46      chs       639:
1.5       mrg       640: void
                    641: uao_detach(uobj)
                    642:        struct uvm_object *uobj;
                    643: {
1.27      chs       644:        simple_lock(&uobj->vmobjlock);
                    645:        uao_detach_locked(uobj);
                    646: }
                    647:
                    648: /*
                    649:  * uao_detach_locked: drop a reference to an aobj
                    650:  *
                    651:  * => aobj must be locked, and is unlocked (or freed) upon return.
                    652:  * this needs to be separate from the normal routine
                    653:  * since sometimes we need to detach from an aobj when
                    654:  * it's already locked.
                    655:  */
1.46      chs       656:
1.27      chs       657: void
                    658: uao_detach_locked(uobj)
                    659:        struct uvm_object *uobj;
                    660: {
1.5       mrg       661:        struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.46      chs       662:        struct vm_page *pg;
1.5       mrg       663:        UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist);
1.1       mrg       664:
1.5       mrg       665:        /*
                    666:         * detaching from kernel_object is a noop.
                    667:         */
1.46      chs       668:
1.27      chs       669:        if (UVM_OBJ_IS_KERN_OBJECT(uobj)) {
                    670:                simple_unlock(&uobj->vmobjlock);
1.5       mrg       671:                return;
1.27      chs       672:        }
1.5       mrg       673:
                    674:        UVMHIST_LOG(maphist,"  (uobj=0x%x)  ref=%d", uobj,uobj->uo_refs,0,0);
1.46      chs       675:        uobj->uo_refs--;
                    676:        if (uobj->uo_refs) {
1.5       mrg       677:                simple_unlock(&uobj->vmobjlock);
                    678:                UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
                    679:                return;
                    680:        }
                    681:
                    682:        /*
                    683:         * remove the aobj from the global list.
                    684:         */
1.46      chs       685:
1.5       mrg       686:        simple_lock(&uao_list_lock);
                    687:        LIST_REMOVE(aobj, u_list);
                    688:        simple_unlock(&uao_list_lock);
                    689:
                    690:        /*
1.46      chs       691:         * free all the pages left in the aobj.  for each page,
                    692:         * when the page is no longer busy (and thus after any disk i/o that
                    693:         * it's involved in is complete), release any swap resources and
                    694:         * free the page itself.
1.5       mrg       695:         */
1.46      chs       696:
                    697:        uvm_lock_pageq();
                    698:        while ((pg = TAILQ_FIRST(&uobj->memq)) != NULL) {
                    699:                pmap_page_protect(pg, VM_PROT_NONE);
1.5       mrg       700:                if (pg->flags & PG_BUSY) {
1.46      chs       701:                        pg->flags |= PG_WANTED;
                    702:                        uvm_unlock_pageq();
                    703:                        UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, FALSE,
                    704:                            "uao_det", 0);
                    705:                        simple_lock(&uobj->vmobjlock);
                    706:                        uvm_lock_pageq();
1.5       mrg       707:                        continue;
                    708:                }
1.18      chs       709:                uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
1.5       mrg       710:                uvm_pagefree(pg);
                    711:        }
1.46      chs       712:        uvm_unlock_pageq();
1.1       mrg       713:
1.5       mrg       714:        /*
1.46      chs       715:         * finally, free the aobj itself.
1.5       mrg       716:         */
1.1       mrg       717:
1.5       mrg       718:        uao_free(aobj);
                    719: }
1.1       mrg       720:
                    721: /*
1.46      chs       722:  * uao_put: flush pages out of a uvm object
1.22      thorpej   723:  *
                    724:  * => object should be locked by caller.  we may _unlock_ the object
                    725:  *     if (and only if) we need to clean a page (PGO_CLEANIT).
                    726:  *     XXXJRT Currently, however, we don't.  In the case of cleaning
                    727:  *     XXXJRT a page, we simply just deactivate it.  Should probably
                    728:  *     XXXJRT handle this better, in the future (although "flushing"
                    729:  *     XXXJRT anonymous memory isn't terribly important).
                    730:  * => if PGO_CLEANIT is not set, then we will neither unlock the object
                    731:  *     or block.
                    732:  * => if PGO_ALLPAGE is set, then all pages in the object are valid targets
                    733:  *     for flushing.
                    734:  * => NOTE: we rely on the fact that the object's memq is a TAILQ and
                    735:  *     that new pages are inserted on the tail end of the list.  thus,
                    736:  *     we can make a complete pass through the object in one go by starting
                    737:  *     at the head and working towards the tail (new pages are put in
                    738:  *     front of us).
                    739:  * => NOTE: we are allowed to lock the page queues, so the caller
                    740:  *     must not be holding the lock on them [e.g. pagedaemon had
                    741:  *     better not call us with the queues locked]
                    742:  * => we return TRUE unless we encountered some sort of I/O error
                    743:  *     XXXJRT currently never happens, as we never directly initiate
                    744:  *     XXXJRT I/O
                    745:  *
                    746:  * note on page traversal:
                    747:  *     we can traverse the pages in an object either by going down the
                    748:  *     linked list in "uobj->memq", or we can go over the address range
                    749:  *     by page doing hash table lookups for each address.  depending
                    750:  *     on how many pages are in the object it may be cheaper to do one
                    751:  *     or the other.  we set "by_list" to true if we are using memq.
                    752:  *     if the cost of a hash lookup was equal to the cost of the list
                    753:  *     traversal we could compare the number of pages in the start->stop
                    754:  *     range to the total number of pages in the object.  however, it
                    755:  *     seems that a hash table lookup is more expensive than the linked
                    756:  *     list traversal, so we multiply the number of pages in the
                    757:  *     start->stop range by a penalty which we define below.
1.1       mrg       758:  */
1.22      thorpej   759:
1.46      chs       760: int
                    761: uao_put(uobj, start, stop, flags)
1.5       mrg       762:        struct uvm_object *uobj;
1.28      kleink    763:        voff_t start, stop;
1.5       mrg       764:        int flags;
                    765: {
1.46      chs       766:        struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
                    767:        struct vm_page *pg, *nextpg;
                    768:        boolean_t by_list;
1.28      kleink    769:        voff_t curoff;
1.46      chs       770:        UVMHIST_FUNC("uao_put"); UVMHIST_CALLED(maphist);
1.22      thorpej   771:
1.46      chs       772:        curoff = 0;
1.22      thorpej   773:        if (flags & PGO_ALLPAGES) {
                    774:                start = 0;
                    775:                stop = aobj->u_pages << PAGE_SHIFT;
                    776:                by_list = TRUE;         /* always go by the list */
                    777:        } else {
                    778:                start = trunc_page(start);
                    779:                stop = round_page(stop);
                    780:                if (stop > (aobj->u_pages << PAGE_SHIFT)) {
                    781:                        printf("uao_flush: strange, got an out of range "
                    782:                            "flush (fixed)\n");
                    783:                        stop = aobj->u_pages << PAGE_SHIFT;
                    784:                }
                    785:                by_list = (uobj->uo_npages <=
1.46      chs       786:                    ((stop - start) >> PAGE_SHIFT) * UVM_PAGE_HASH_PENALTY);
1.22      thorpej   787:        }
                    788:        UVMHIST_LOG(maphist,
                    789:            " flush start=0x%lx, stop=0x%x, by_list=%d, flags=0x%x",
                    790:            start, stop, by_list, flags);
1.1       mrg       791:
1.5       mrg       792:        /*
1.22      thorpej   793:         * Don't need to do any work here if we're not freeing
                    794:         * or deactivating pages.
                    795:         */
1.46      chs       796:
1.22      thorpej   797:        if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
1.46      chs       798:                simple_unlock(&uobj->vmobjlock);
                    799:                return 0;
1.22      thorpej   800:        }
                    801:
1.5       mrg       802:        /*
1.46      chs       803:         * now do it.  note: we must update nextpg in the body of loop or we
                    804:         * will get stuck.  we need to use nextpg because we may free "pg"
1.22      thorpej   805:         * before doing the next loop.
1.21      thorpej   806:         */
1.22      thorpej   807:
                    808:        if (by_list) {
1.46      chs       809:                pg = TAILQ_FIRST(&uobj->memq);
1.22      thorpej   810:        } else {
                    811:                curoff = start;
1.46      chs       812:                pg = uvm_pagelookup(uobj, curoff);
1.22      thorpej   813:        }
                    814:
1.46      chs       815:        nextpg = NULL;
                    816:        uvm_lock_pageq();
1.22      thorpej   817:
                    818:        /* locked: both page queues and uobj */
1.46      chs       819:        for ( ; (by_list && pg != NULL) ||
                    820:            (!by_list && curoff < stop) ; pg = nextpg) {
1.22      thorpej   821:                if (by_list) {
1.46      chs       822:                        nextpg = TAILQ_NEXT(pg, listq);
                    823:                        if (pg->offset < start || pg->offset >= stop)
1.22      thorpej   824:                                continue;
                    825:                } else {
                    826:                        curoff += PAGE_SIZE;
                    827:                        if (curoff < stop)
1.46      chs       828:                                nextpg = uvm_pagelookup(uobj, curoff);
                    829:                        if (pg == NULL)
1.22      thorpej   830:                                continue;
                    831:                }
1.46      chs       832:                switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
1.41      chs       833:
1.22      thorpej   834:                /*
                    835:                 * XXX In these first 3 cases, we always just
                    836:                 * XXX deactivate the page.  We may want to
                    837:                 * XXX handle the different cases more specifically
                    838:                 * XXX in the future.
                    839:                 */
1.46      chs       840:
1.22      thorpej   841:                case PGO_CLEANIT|PGO_FREE:
                    842:                case PGO_CLEANIT|PGO_DEACTIVATE:
                    843:                case PGO_DEACTIVATE:
1.25      thorpej   844:  deactivate_it:
1.22      thorpej   845:                        /* skip the page if it's loaned or wired */
1.46      chs       846:                        if (pg->loan_count != 0 || pg->wire_count != 0)
1.22      thorpej   847:                                continue;
                    848:
                    849:                        /* ...and deactivate the page. */
1.46      chs       850:                        pmap_clear_reference(pg);
                    851:                        uvm_pagedeactivate(pg);
1.22      thorpej   852:                        continue;
                    853:
                    854:                case PGO_FREE:
1.46      chs       855:
1.25      thorpej   856:                        /*
                    857:                         * If there are multiple references to
                    858:                         * the object, just deactivate the page.
                    859:                         */
1.46      chs       860:
1.25      thorpej   861:                        if (uobj->uo_refs > 1)
                    862:                                goto deactivate_it;
                    863:
1.22      thorpej   864:                        /* XXX skip the page if it's loaned or wired */
1.46      chs       865:                        if (pg->loan_count != 0 || pg->wire_count != 0)
1.22      thorpej   866:                                continue;
                    867:
                    868:                        /*
1.46      chs       869:                         * wait if the page is busy, then free the swap slot
                    870:                         * and the page.
1.22      thorpej   871:                         */
1.46      chs       872:
                    873:                        pmap_page_protect(pg, VM_PROT_NONE);
                    874:                        while (pg->flags & PG_BUSY) {
                    875:                                pg->flags |= PG_WANTED;
                    876:                                uvm_unlock_pageq();
                    877:                                UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, 0,
                    878:                                    "uao_put", 0);
                    879:                                simple_lock(&uobj->vmobjlock);
                    880:                                uvm_lock_pageq();
1.22      thorpej   881:                        }
1.46      chs       882:                        uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
                    883:                        uvm_pagefree(pg);
1.22      thorpej   884:                        continue;
                    885:                }
                    886:        }
                    887:        uvm_unlock_pageq();
1.46      chs       888:        simple_unlock(&uobj->vmobjlock);
                    889:        return 0;
1.1       mrg       890: }
                    891:
                    892: /*
                    893:  * uao_get: fetch me a page
                    894:  *
                    895:  * we have three cases:
                    896:  * 1: page is resident     -> just return the page.
                    897:  * 2: page is zero-fill    -> allocate a new page and zero it.
                    898:  * 3: page is swapped out  -> fetch the page from swap.
                    899:  *
                    900:  * cases 1 and 2 can be handled with PGO_LOCKED, case 3 cannot.
                    901:  * so, if the "center" page hits case 3 (or any page, with PGO_ALLPAGES),
1.40      chs       902:  * then we will need to return EBUSY.
1.1       mrg       903:  *
                    904:  * => prefer map unlocked (not required)
                    905:  * => object must be locked!  we will _unlock_ it before starting any I/O.
                    906:  * => flags: PGO_ALLPAGES: get all of the pages
                    907:  *           PGO_LOCKED: fault data structures are locked
                    908:  * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
                    909:  * => NOTE: caller must check for released pages!!
                    910:  */
1.46      chs       911:
1.5       mrg       912: static int
                    913: uao_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags)
                    914:        struct uvm_object *uobj;
1.28      kleink    915:        voff_t offset;
1.5       mrg       916:        struct vm_page **pps;
                    917:        int *npagesp;
                    918:        int centeridx, advice, flags;
                    919:        vm_prot_t access_type;
                    920: {
                    921:        struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.28      kleink    922:        voff_t current_offset;
1.43      chs       923:        struct vm_page *ptmp;
1.46      chs       924:        int lcv, gotpages, maxpages, swslot, error, pageidx;
1.5       mrg       925:        boolean_t done;
                    926:        UVMHIST_FUNC("uao_get"); UVMHIST_CALLED(pdhist);
                    927:
1.27      chs       928:        UVMHIST_LOG(pdhist, "aobj=%p offset=%d, flags=%d",
                    929:                    aobj, offset, flags,0);
1.37      chs       930:
1.5       mrg       931:        /*
                    932:         * get number of pages
                    933:         */
1.46      chs       934:
1.5       mrg       935:        maxpages = *npagesp;
                    936:
                    937:        /*
                    938:         * step 1: handled the case where fault data structures are locked.
                    939:         */
1.1       mrg       940:
1.5       mrg       941:        if (flags & PGO_LOCKED) {
1.46      chs       942:
1.5       mrg       943:                /*
                    944:                 * step 1a: get pages that are already resident.   only do
                    945:                 * this if the data structures are locked (i.e. the first
                    946:                 * time through).
                    947:                 */
                    948:
                    949:                done = TRUE;    /* be optimistic */
                    950:                gotpages = 0;   /* # of pages we got so far */
                    951:                for (lcv = 0, current_offset = offset ; lcv < maxpages ;
                    952:                    lcv++, current_offset += PAGE_SIZE) {
                    953:                        /* do we care about this page?  if not, skip it */
                    954:                        if (pps[lcv] == PGO_DONTCARE)
                    955:                                continue;
                    956:                        ptmp = uvm_pagelookup(uobj, current_offset);
                    957:
                    958:                        /*
1.30      thorpej   959:                         * if page is new, attempt to allocate the page,
                    960:                         * zero-fill'd.
1.5       mrg       961:                         */
1.46      chs       962:
                    963:                        if (ptmp == NULL && uao_find_swslot(&aobj->u_obj,
1.15      chs       964:                            current_offset >> PAGE_SHIFT) == 0) {
1.5       mrg       965:                                ptmp = uvm_pagealloc(uobj, current_offset,
1.30      thorpej   966:                                    NULL, UVM_PGA_ZERO);
1.5       mrg       967:                                if (ptmp) {
                    968:                                        /* new page */
1.47      chs       969:                                        ptmp->flags &= ~(PG_FAKE);
1.5       mrg       970:                                        ptmp->pqflags |= PQ_AOBJ;
1.47      chs       971:                                        goto gotpage;
1.5       mrg       972:                                }
                    973:                        }
                    974:
                    975:                        /*
1.46      chs       976:                         * to be useful must get a non-busy page
1.5       mrg       977:                         */
1.46      chs       978:
                    979:                        if (ptmp == NULL || (ptmp->flags & PG_BUSY) != 0) {
1.5       mrg       980:                                if (lcv == centeridx ||
                    981:                                    (flags & PGO_ALLPAGES) != 0)
                    982:                                        /* need to do a wait or I/O! */
1.41      chs       983:                                        done = FALSE;
1.5       mrg       984:                                        continue;
                    985:                        }
                    986:
                    987:                        /*
                    988:                         * useful page: busy/lock it and plug it in our
                    989:                         * result array
                    990:                         */
1.46      chs       991:
1.5       mrg       992:                        /* caller must un-busy this page */
1.41      chs       993:                        ptmp->flags |= PG_BUSY;
1.5       mrg       994:                        UVM_PAGE_OWN(ptmp, "uao_get1");
1.47      chs       995: gotpage:
1.5       mrg       996:                        pps[lcv] = ptmp;
                    997:                        gotpages++;
1.46      chs       998:                }
1.5       mrg       999:
                   1000:                /*
                   1001:                 * step 1b: now we've either done everything needed or we
                   1002:                 * to unlock and do some waiting or I/O.
                   1003:                 */
                   1004:
                   1005:                UVMHIST_LOG(pdhist, "<- done (done=%d)", done, 0,0,0);
                   1006:                *npagesp = gotpages;
                   1007:                if (done)
1.46      chs      1008:                        return 0;
1.5       mrg      1009:                else
1.46      chs      1010:                        return EBUSY;
1.1       mrg      1011:        }
                   1012:
1.5       mrg      1013:        /*
                   1014:         * step 2: get non-resident or busy pages.
                   1015:         * object is locked.   data structures are unlocked.
                   1016:         */
                   1017:
                   1018:        for (lcv = 0, current_offset = offset ; lcv < maxpages ;
                   1019:            lcv++, current_offset += PAGE_SIZE) {
1.27      chs      1020:
1.5       mrg      1021:                /*
                   1022:                 * - skip over pages we've already gotten or don't want
                   1023:                 * - skip over pages we don't _have_ to get
                   1024:                 */
1.27      chs      1025:
1.5       mrg      1026:                if (pps[lcv] != NULL ||
                   1027:                    (lcv != centeridx && (flags & PGO_ALLPAGES) == 0))
                   1028:                        continue;
                   1029:
1.27      chs      1030:                pageidx = current_offset >> PAGE_SHIFT;
                   1031:
1.5       mrg      1032:                /*
                   1033:                 * we have yet to locate the current page (pps[lcv]).   we
                   1034:                 * first look for a page that is already at the current offset.
                   1035:                 * if we find a page, we check to see if it is busy or
                   1036:                 * released.  if that is the case, then we sleep on the page
                   1037:                 * until it is no longer busy or released and repeat the lookup.
                   1038:                 * if the page we found is neither busy nor released, then we
                   1039:                 * busy it (so we own it) and plug it into pps[lcv].   this
                   1040:                 * 'break's the following while loop and indicates we are
                   1041:                 * ready to move on to the next page in the "lcv" loop above.
                   1042:                 *
                   1043:                 * if we exit the while loop with pps[lcv] still set to NULL,
                   1044:                 * then it means that we allocated a new busy/fake/clean page
                   1045:                 * ptmp in the object and we need to do I/O to fill in the data.
                   1046:                 */
                   1047:
                   1048:                /* top of "pps" while loop */
                   1049:                while (pps[lcv] == NULL) {
                   1050:                        /* look for a resident page */
                   1051:                        ptmp = uvm_pagelookup(uobj, current_offset);
                   1052:
                   1053:                        /* not resident?   allocate one now (if we can) */
                   1054:                        if (ptmp == NULL) {
                   1055:
                   1056:                                ptmp = uvm_pagealloc(uobj, current_offset,
1.19      chs      1057:                                    NULL, 0);
1.5       mrg      1058:
                   1059:                                /* out of RAM? */
                   1060:                                if (ptmp == NULL) {
                   1061:                                        simple_unlock(&uobj->vmobjlock);
                   1062:                                        UVMHIST_LOG(pdhist,
                   1063:                                            "sleeping, ptmp == NULL\n",0,0,0,0);
                   1064:                                        uvm_wait("uao_getpage");
                   1065:                                        simple_lock(&uobj->vmobjlock);
1.41      chs      1066:                                        continue;
1.5       mrg      1067:                                }
                   1068:
                   1069:                                /*
                   1070:                                 * safe with PQ's unlocked: because we just
                   1071:                                 * alloc'd the page
                   1072:                                 */
1.46      chs      1073:
1.5       mrg      1074:                                ptmp->pqflags |= PQ_AOBJ;
                   1075:
1.41      chs      1076:                                /*
1.5       mrg      1077:                                 * got new page ready for I/O.  break pps while
                   1078:                                 * loop.  pps[lcv] is still NULL.
                   1079:                                 */
1.46      chs      1080:
1.5       mrg      1081:                                break;
                   1082:                        }
                   1083:
                   1084:                        /* page is there, see if we need to wait on it */
1.46      chs      1085:                        if ((ptmp->flags & PG_BUSY) != 0) {
1.5       mrg      1086:                                ptmp->flags |= PG_WANTED;
                   1087:                                UVMHIST_LOG(pdhist,
                   1088:                                    "sleeping, ptmp->flags 0x%x\n",
                   1089:                                    ptmp->flags,0,0,0);
1.23      thorpej  1090:                                UVM_UNLOCK_AND_WAIT(ptmp, &uobj->vmobjlock,
                   1091:                                    FALSE, "uao_get", 0);
1.5       mrg      1092:                                simple_lock(&uobj->vmobjlock);
1.46      chs      1093:                                continue;
1.5       mrg      1094:                        }
1.41      chs      1095:
                   1096:                        /*
1.5       mrg      1097:                         * if we get here then the page has become resident and
                   1098:                         * unbusy between steps 1 and 2.  we busy it now (so we
                   1099:                         * own it) and set pps[lcv] (so that we exit the while
                   1100:                         * loop).
                   1101:                         */
1.46      chs      1102:
1.5       mrg      1103:                        /* we own it, caller must un-busy */
                   1104:                        ptmp->flags |= PG_BUSY;
                   1105:                        UVM_PAGE_OWN(ptmp, "uao_get2");
                   1106:                        pps[lcv] = ptmp;
                   1107:                }
                   1108:
                   1109:                /*
                   1110:                 * if we own the valid page at the correct offset, pps[lcv] will
                   1111:                 * point to it.   nothing more to do except go to the next page.
                   1112:                 */
1.46      chs      1113:
1.5       mrg      1114:                if (pps[lcv])
                   1115:                        continue;                       /* next lcv */
                   1116:
                   1117:                /*
1.41      chs      1118:                 * we have a "fake/busy/clean" page that we just allocated.
1.5       mrg      1119:                 * do the needed "i/o", either reading from swap or zeroing.
                   1120:                 */
1.46      chs      1121:
                   1122:                swslot = uao_find_swslot(&aobj->u_obj, pageidx);
1.5       mrg      1123:
                   1124:                /*
                   1125:                 * just zero the page if there's nothing in swap.
                   1126:                 */
1.46      chs      1127:
                   1128:                if (swslot == 0) {
                   1129:
1.5       mrg      1130:                        /*
                   1131:                         * page hasn't existed before, just zero it.
                   1132:                         */
1.46      chs      1133:
1.5       mrg      1134:                        uvm_pagezero(ptmp);
1.27      chs      1135:                } else {
1.5       mrg      1136:                        UVMHIST_LOG(pdhist, "pagein from swslot %d",
                   1137:                             swslot, 0,0,0);
                   1138:
                   1139:                        /*
                   1140:                         * page in the swapped-out page.
                   1141:                         * unlock object for i/o, relock when done.
                   1142:                         */
1.46      chs      1143:
1.5       mrg      1144:                        simple_unlock(&uobj->vmobjlock);
1.46      chs      1145:                        error = uvm_swap_get(ptmp, swslot, PGO_SYNCIO);
1.5       mrg      1146:                        simple_lock(&uobj->vmobjlock);
                   1147:
                   1148:                        /*
                   1149:                         * I/O done.  check for errors.
                   1150:                         */
1.46      chs      1151:
                   1152:                        if (error != 0) {
1.5       mrg      1153:                                UVMHIST_LOG(pdhist, "<- done (error=%d)",
1.46      chs      1154:                                    error,0,0,0);
1.5       mrg      1155:                                if (ptmp->flags & PG_WANTED)
1.24      thorpej  1156:                                        wakeup(ptmp);
1.27      chs      1157:
                   1158:                                /*
                   1159:                                 * remove the swap slot from the aobj
                   1160:                                 * and mark the aobj as having no real slot.
                   1161:                                 * don't free the swap slot, thus preventing
                   1162:                                 * it from being used again.
                   1163:                                 */
1.46      chs      1164:
1.27      chs      1165:                                swslot = uao_set_swslot(&aobj->u_obj, pageidx,
                   1166:                                                        SWSLOT_BAD);
1.45      chs      1167:                                if (swslot != -1) {
                   1168:                                        uvm_swap_markbad(swslot, 1);
                   1169:                                }
1.27      chs      1170:
1.5       mrg      1171:                                uvm_lock_pageq();
                   1172:                                uvm_pagefree(ptmp);
                   1173:                                uvm_unlock_pageq();
                   1174:                                simple_unlock(&uobj->vmobjlock);
1.46      chs      1175:                                return error;
1.5       mrg      1176:                        }
                   1177:                }
                   1178:
1.41      chs      1179:                /*
1.5       mrg      1180:                 * we got the page!   clear the fake flag (indicates valid
                   1181:                 * data now in page) and plug into our result array.   note
1.41      chs      1182:                 * that page is still busy.
1.5       mrg      1183:                 *
                   1184:                 * it is the callers job to:
                   1185:                 * => check if the page is released
                   1186:                 * => unbusy the page
                   1187:                 * => activate the page
                   1188:                 */
                   1189:
1.46      chs      1190:                ptmp->flags &= ~PG_FAKE;
1.5       mrg      1191:                pps[lcv] = ptmp;
1.46      chs      1192:        }
1.1       mrg      1193:
                   1194:        /*
1.5       mrg      1195:         * finally, unlock object and return.
                   1196:         */
1.1       mrg      1197:
                   1198:        simple_unlock(&uobj->vmobjlock);
1.5       mrg      1199:        UVMHIST_LOG(pdhist, "<- done (OK)",0,0,0,0);
1.46      chs      1200:        return 0;
1.1       mrg      1201: }
                   1202:
                   1203: /*
1.18      chs      1204:  * uao_dropswap:  release any swap resources from this aobj page.
1.41      chs      1205:  *
1.18      chs      1206:  * => aobj must be locked or have a reference count of 0.
                   1207:  */
                   1208:
                   1209: void
                   1210: uao_dropswap(uobj, pageidx)
                   1211:        struct uvm_object *uobj;
                   1212:        int pageidx;
                   1213: {
                   1214:        int slot;
                   1215:
                   1216:        slot = uao_set_swslot(uobj, pageidx, 0);
                   1217:        if (slot) {
                   1218:                uvm_swap_free(slot, 1);
                   1219:        }
1.27      chs      1220: }
                   1221:
                   1222: /*
                   1223:  * page in every page in every aobj that is paged-out to a range of swslots.
1.41      chs      1224:  *
1.27      chs      1225:  * => nothing should be locked.
                   1226:  * => returns TRUE if pagein was aborted due to lack of memory.
                   1227:  */
1.46      chs      1228:
1.27      chs      1229: boolean_t
                   1230: uao_swap_off(startslot, endslot)
                   1231:        int startslot, endslot;
                   1232: {
                   1233:        struct uvm_aobj *aobj, *nextaobj;
1.46      chs      1234:        boolean_t rv;
1.27      chs      1235:
                   1236:        /*
                   1237:         * walk the list of all aobjs.
                   1238:         */
                   1239:
                   1240: restart:
                   1241:        simple_lock(&uao_list_lock);
                   1242:        for (aobj = LIST_FIRST(&uao_list);
                   1243:             aobj != NULL;
                   1244:             aobj = nextaobj) {
                   1245:
                   1246:                /*
1.46      chs      1247:                 * try to get the object lock, start all over if we fail.
1.27      chs      1248:                 * most of the time we'll get the aobj lock,
                   1249:                 * so this should be a rare case.
                   1250:                 */
1.46      chs      1251:
1.27      chs      1252:                if (!simple_lock_try(&aobj->u_obj.vmobjlock)) {
                   1253:                        simple_unlock(&uao_list_lock);
                   1254:                        goto restart;
                   1255:                }
                   1256:
                   1257:                /*
                   1258:                 * add a ref to the aobj so it doesn't disappear
                   1259:                 * while we're working.
                   1260:                 */
1.46      chs      1261:
1.27      chs      1262:                uao_reference_locked(&aobj->u_obj);
                   1263:
                   1264:                /*
                   1265:                 * now it's safe to unlock the uao list.
                   1266:                 */
1.46      chs      1267:
1.27      chs      1268:                simple_unlock(&uao_list_lock);
                   1269:
                   1270:                /*
                   1271:                 * page in any pages in the swslot range.
                   1272:                 * if there's an error, abort and return the error.
                   1273:                 */
1.46      chs      1274:
1.27      chs      1275:                rv = uao_pagein(aobj, startslot, endslot);
                   1276:                if (rv) {
                   1277:                        uao_detach_locked(&aobj->u_obj);
                   1278:                        return rv;
                   1279:                }
                   1280:
                   1281:                /*
                   1282:                 * we're done with this aobj.
                   1283:                 * relock the list and drop our ref on the aobj.
                   1284:                 */
1.46      chs      1285:
1.27      chs      1286:                simple_lock(&uao_list_lock);
                   1287:                nextaobj = LIST_NEXT(aobj, u_list);
                   1288:                uao_detach_locked(&aobj->u_obj);
                   1289:        }
                   1290:
                   1291:        /*
                   1292:         * done with traversal, unlock the list
                   1293:         */
                   1294:        simple_unlock(&uao_list_lock);
                   1295:        return FALSE;
                   1296: }
                   1297:
                   1298:
                   1299: /*
                   1300:  * page in any pages from aobj in the given range.
                   1301:  *
                   1302:  * => aobj must be locked and is returned locked.
                   1303:  * => returns TRUE if pagein was aborted due to lack of memory.
                   1304:  */
                   1305: static boolean_t
                   1306: uao_pagein(aobj, startslot, endslot)
                   1307:        struct uvm_aobj *aobj;
                   1308:        int startslot, endslot;
                   1309: {
                   1310:        boolean_t rv;
                   1311:
                   1312:        if (UAO_USES_SWHASH(aobj)) {
                   1313:                struct uao_swhash_elt *elt;
                   1314:                int bucket;
                   1315:
                   1316: restart:
                   1317:                for (bucket = aobj->u_swhashmask; bucket >= 0; bucket--) {
                   1318:                        for (elt = LIST_FIRST(&aobj->u_swhash[bucket]);
                   1319:                             elt != NULL;
                   1320:                             elt = LIST_NEXT(elt, list)) {
                   1321:                                int i;
                   1322:
                   1323:                                for (i = 0; i < UAO_SWHASH_CLUSTER_SIZE; i++) {
                   1324:                                        int slot = elt->slots[i];
                   1325:
                   1326:                                        /*
                   1327:                                         * if the slot isn't in range, skip it.
                   1328:                                         */
1.46      chs      1329:
1.41      chs      1330:                                        if (slot < startslot ||
1.27      chs      1331:                                            slot >= endslot) {
                   1332:                                                continue;
                   1333:                                        }
                   1334:
                   1335:                                        /*
                   1336:                                         * process the page,
                   1337:                                         * the start over on this object
                   1338:                                         * since the swhash elt
                   1339:                                         * may have been freed.
                   1340:                                         */
1.46      chs      1341:
1.27      chs      1342:                                        rv = uao_pagein_page(aobj,
                   1343:                                          UAO_SWHASH_ELT_PAGEIDX_BASE(elt) + i);
                   1344:                                        if (rv) {
                   1345:                                                return rv;
                   1346:                                        }
                   1347:                                        goto restart;
                   1348:                                }
                   1349:                        }
                   1350:                }
                   1351:        } else {
                   1352:                int i;
                   1353:
                   1354:                for (i = 0; i < aobj->u_pages; i++) {
                   1355:                        int slot = aobj->u_swslots[i];
                   1356:
                   1357:                        /*
                   1358:                         * if the slot isn't in range, skip it
                   1359:                         */
1.46      chs      1360:
1.27      chs      1361:                        if (slot < startslot || slot >= endslot) {
                   1362:                                continue;
                   1363:                        }
                   1364:
                   1365:                        /*
                   1366:                         * process the page.
                   1367:                         */
1.46      chs      1368:
1.27      chs      1369:                        rv = uao_pagein_page(aobj, i);
                   1370:                        if (rv) {
                   1371:                                return rv;
                   1372:                        }
                   1373:                }
                   1374:        }
                   1375:
                   1376:        return FALSE;
                   1377: }
                   1378:
                   1379: /*
                   1380:  * page in a page from an aobj.  used for swap_off.
                   1381:  * returns TRUE if pagein was aborted due to lack of memory.
                   1382:  *
                   1383:  * => aobj must be locked and is returned locked.
                   1384:  */
1.46      chs      1385:
1.27      chs      1386: static boolean_t
                   1387: uao_pagein_page(aobj, pageidx)
                   1388:        struct uvm_aobj *aobj;
                   1389:        int pageidx;
                   1390: {
                   1391:        struct vm_page *pg;
                   1392:        int rv, slot, npages;
                   1393:
                   1394:        pg = NULL;
                   1395:        npages = 1;
                   1396:        /* locked: aobj */
                   1397:        rv = uao_get(&aobj->u_obj, pageidx << PAGE_SHIFT,
                   1398:                     &pg, &npages, 0, VM_PROT_READ|VM_PROT_WRITE, 0, 0);
                   1399:        /* unlocked: aobj */
                   1400:
                   1401:        /*
                   1402:         * relock and finish up.
                   1403:         */
1.46      chs      1404:
1.27      chs      1405:        simple_lock(&aobj->u_obj.vmobjlock);
                   1406:        switch (rv) {
1.40      chs      1407:        case 0:
1.27      chs      1408:                break;
                   1409:
1.40      chs      1410:        case EIO:
                   1411:        case ERESTART:
1.46      chs      1412:
1.27      chs      1413:                /*
                   1414:                 * nothing more to do on errors.
1.40      chs      1415:                 * ERESTART can only mean that the anon was freed,
1.27      chs      1416:                 * so again there's nothing to do.
                   1417:                 */
1.46      chs      1418:
1.27      chs      1419:                return FALSE;
                   1420:        }
                   1421:
                   1422:        /*
                   1423:         * ok, we've got the page now.
                   1424:         * mark it as dirty, clear its swslot and un-busy it.
                   1425:         */
1.46      chs      1426:
1.27      chs      1427:        slot = uao_set_swslot(&aobj->u_obj, pageidx, 0);
                   1428:        uvm_swap_free(slot, 1);
                   1429:        pg->flags &= ~(PG_BUSY|PG_CLEAN|PG_FAKE);
                   1430:        UVM_PAGE_OWN(pg, NULL);
                   1431:
                   1432:        /*
1.46      chs      1433:         * deactivate the page (to make sure it's on a page queue).
1.27      chs      1434:         */
1.46      chs      1435:
1.27      chs      1436:        uvm_lock_pageq();
                   1437:        uvm_pagedeactivate(pg);
                   1438:        uvm_unlock_pageq();
                   1439:        return FALSE;
1.1       mrg      1440: }

CVSweb <webmaster@jp.NetBSD.org>