[BACK]Return to uvm_aobj.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / uvm

Annotation of src/sys/uvm/uvm_aobj.c, Revision 1.39.2.4

1.39.2.4! nathanw     1: /*     $NetBSD: uvm_aobj.c,v 1.39.2.3 2001/08/24 00:13:33 nathanw Exp $        */
1.6       mrg         2:
1.7       chs         3: /*
                      4:  * Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
                      5:  *                    Washington University.
                      6:  * All rights reserved.
                      7:  *
                      8:  * Redistribution and use in source and binary forms, with or without
                      9:  * modification, are permitted provided that the following conditions
                     10:  * are met:
                     11:  * 1. Redistributions of source code must retain the above copyright
                     12:  *    notice, this list of conditions and the following disclaimer.
                     13:  * 2. Redistributions in binary form must reproduce the above copyright
                     14:  *    notice, this list of conditions and the following disclaimer in the
                     15:  *    documentation and/or other materials provided with the distribution.
                     16:  * 3. All advertising materials mentioning features or use of this software
                     17:  *    must display the following acknowledgement:
                     18:  *      This product includes software developed by Charles D. Cranor and
                     19:  *      Washington University.
                     20:  * 4. The name of the author may not be used to endorse or promote products
                     21:  *    derived from this software without specific prior written permission.
                     22:  *
                     23:  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
                     24:  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
                     25:  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
                     26:  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
                     27:  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
                     28:  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
                     29:  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
                     30:  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
                     31:  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
                     32:  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
                     33:  *
1.4       mrg        34:  * from: Id: uvm_aobj.c,v 1.1.2.5 1998/02/06 05:14:38 chs Exp
                     35:  */
1.7       chs        36: /*
                     37:  * uvm_aobj.c: anonymous memory uvm_object pager
                     38:  *
                     39:  * author: Chuck Silvers <chuq@chuq.com>
                     40:  * started: Jan-1998
                     41:  *
                     42:  * - design mostly from Chuck Cranor
                     43:  */
                     44:
                     45: #include "opt_uvmhist.h"
1.1       mrg        46:
                     47: #include <sys/param.h>
                     48: #include <sys/systm.h>
                     49: #include <sys/proc.h>
                     50: #include <sys/malloc.h>
1.37      chs        51: #include <sys/kernel.h>
1.12      thorpej    52: #include <sys/pool.h>
1.27      chs        53: #include <sys/kernel.h>
1.1       mrg        54:
                     55: #include <uvm/uvm.h>
                     56:
                     57: /*
                     58:  * an aobj manages anonymous-memory backed uvm_objects.   in addition
                     59:  * to keeping the list of resident pages, it also keeps a list of
                     60:  * allocated swap blocks.  depending on the size of the aobj this list
                     61:  * of allocated swap blocks is either stored in an array (small objects)
                     62:  * or in a hash table (large objects).
                     63:  */
                     64:
                     65: /*
                     66:  * local structures
                     67:  */
                     68:
                     69: /*
                     70:  * for hash tables, we break the address space of the aobj into blocks
                     71:  * of UAO_SWHASH_CLUSTER_SIZE pages.   we require the cluster size to
                     72:  * be a power of two.
                     73:  */
                     74:
                     75: #define UAO_SWHASH_CLUSTER_SHIFT 4
                     76: #define UAO_SWHASH_CLUSTER_SIZE (1 << UAO_SWHASH_CLUSTER_SHIFT)
                     77:
                     78: /* get the "tag" for this page index */
                     79: #define UAO_SWHASH_ELT_TAG(PAGEIDX) \
                     80:        ((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT)
                     81:
                     82: /* given an ELT and a page index, find the swap slot */
                     83: #define UAO_SWHASH_ELT_PAGESLOT(ELT, PAGEIDX) \
                     84:        ((ELT)->slots[(PAGEIDX) & (UAO_SWHASH_CLUSTER_SIZE - 1)])
                     85:
                     86: /* given an ELT, return its pageidx base */
                     87: #define UAO_SWHASH_ELT_PAGEIDX_BASE(ELT) \
                     88:        ((ELT)->tag << UAO_SWHASH_CLUSTER_SHIFT)
                     89:
                     90: /*
                     91:  * the swhash hash function
                     92:  */
1.39.2.4! nathanw    93:
1.1       mrg        94: #define UAO_SWHASH_HASH(AOBJ, PAGEIDX) \
                     95:        (&(AOBJ)->u_swhash[(((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT) \
                     96:                            & (AOBJ)->u_swhashmask)])
                     97:
                     98: /*
                     99:  * the swhash threshhold determines if we will use an array or a
                    100:  * hash table to store the list of allocated swap blocks.
                    101:  */
                    102:
                    103: #define UAO_SWHASH_THRESHOLD (UAO_SWHASH_CLUSTER_SIZE * 4)
                    104: #define UAO_USES_SWHASH(AOBJ) \
                    105:        ((AOBJ)->u_pages > UAO_SWHASH_THRESHOLD)        /* use hash? */
                    106:
                    107: /*
1.3       chs       108:  * the number of buckets in a swhash, with an upper bound
1.1       mrg       109:  */
1.39.2.4! nathanw   110:
1.1       mrg       111: #define UAO_SWHASH_MAXBUCKETS 256
                    112: #define UAO_SWHASH_BUCKETS(AOBJ) \
1.39.2.4! nathanw   113:        (MIN((AOBJ)->u_pages >> UAO_SWHASH_CLUSTER_SHIFT, \
1.1       mrg       114:             UAO_SWHASH_MAXBUCKETS))
                    115:
                    116:
                    117: /*
                    118:  * uao_swhash_elt: when a hash table is being used, this structure defines
                    119:  * the format of an entry in the bucket list.
                    120:  */
                    121:
                    122: struct uao_swhash_elt {
1.5       mrg       123:        LIST_ENTRY(uao_swhash_elt) list;        /* the hash list */
1.28      kleink    124:        voff_t tag;                             /* our 'tag' */
1.5       mrg       125:        int count;                              /* our number of active slots */
                    126:        int slots[UAO_SWHASH_CLUSTER_SIZE];     /* the slots */
1.1       mrg       127: };
                    128:
                    129: /*
                    130:  * uao_swhash: the swap hash table structure
                    131:  */
                    132:
                    133: LIST_HEAD(uao_swhash, uao_swhash_elt);
                    134:
1.12      thorpej   135: /*
                    136:  * uao_swhash_elt_pool: pool of uao_swhash_elt structures
                    137:  */
                    138:
                    139: struct pool uao_swhash_elt_pool;
1.1       mrg       140:
                    141: /*
                    142:  * uvm_aobj: the actual anon-backed uvm_object
                    143:  *
                    144:  * => the uvm_object is at the top of the structure, this allows
1.39.2.4! nathanw   145:  *   (struct uvm_aobj *) == (struct uvm_object *)
1.1       mrg       146:  * => only one of u_swslots and u_swhash is used in any given aobj
                    147:  */
                    148:
                    149: struct uvm_aobj {
1.5       mrg       150:        struct uvm_object u_obj; /* has: lock, pgops, memq, #pages, #refs */
1.11      drochner  151:        int u_pages;             /* number of pages in entire object */
1.5       mrg       152:        int u_flags;             /* the flags (see uvm_aobj.h) */
                    153:        int *u_swslots;          /* array of offset->swapslot mappings */
                    154:                                 /*
                    155:                                  * hashtable of offset->swapslot mappings
                    156:                                  * (u_swhash is an array of bucket heads)
                    157:                                  */
                    158:        struct uao_swhash *u_swhash;
                    159:        u_long u_swhashmask;            /* mask for hashtable */
                    160:        LIST_ENTRY(uvm_aobj) u_list;    /* global list of aobjs */
1.1       mrg       161: };
                    162:
                    163: /*
1.12      thorpej   164:  * uvm_aobj_pool: pool of uvm_aobj structures
                    165:  */
                    166:
                    167: struct pool uvm_aobj_pool;
                    168:
                    169: /*
1.1       mrg       170:  * local functions
                    171:  */
                    172:
1.39.2.4! nathanw   173: static struct uao_swhash_elt *uao_find_swhash_elt
        !           174:     __P((struct uvm_aobj *, int, boolean_t));
        !           175:
        !           176: static void    uao_free __P((struct uvm_aobj *));
        !           177: static int     uao_get __P((struct uvm_object *, voff_t, struct vm_page **,
        !           178:                    int *, int, vm_prot_t, int, int));
        !           179: static boolean_t uao_put __P((struct uvm_object *, voff_t, voff_t, int));
        !           180: static boolean_t uao_pagein __P((struct uvm_aobj *, int, int));
        !           181: static boolean_t uao_pagein_page __P((struct uvm_aobj *, int));
1.1       mrg       182:
                    183: /*
                    184:  * aobj_pager
1.39.2.2  nathanw   185:  *
1.1       mrg       186:  * note that some functions (e.g. put) are handled elsewhere
                    187:  */
                    188:
                    189: struct uvm_pagerops aobj_pager = {
1.27      chs       190:        NULL,                   /* init */
1.5       mrg       191:        uao_reference,          /* reference */
                    192:        uao_detach,             /* detach */
                    193:        NULL,                   /* fault */
                    194:        uao_get,                /* get */
1.39.2.4! nathanw   195:        uao_put,                /* flush */
1.1       mrg       196: };
                    197:
                    198: /*
                    199:  * uao_list: global list of active aobjs, locked by uao_list_lock
                    200:  */
                    201:
                    202: static LIST_HEAD(aobjlist, uvm_aobj) uao_list;
1.39.2.2  nathanw   203: static struct simplelock uao_list_lock;
1.1       mrg       204:
                    205: /*
                    206:  * functions
                    207:  */
                    208:
                    209: /*
                    210:  * hash table/array related functions
                    211:  */
                    212:
                    213: /*
                    214:  * uao_find_swhash_elt: find (or create) a hash table entry for a page
                    215:  * offset.
                    216:  *
                    217:  * => the object should be locked by the caller
                    218:  */
                    219:
1.5       mrg       220: static struct uao_swhash_elt *
                    221: uao_find_swhash_elt(aobj, pageidx, create)
                    222:        struct uvm_aobj *aobj;
                    223:        int pageidx;
                    224:        boolean_t create;
                    225: {
                    226:        struct uao_swhash *swhash;
                    227:        struct uao_swhash_elt *elt;
1.28      kleink    228:        voff_t page_tag;
1.1       mrg       229:
1.39.2.3  nathanw   230:        swhash = UAO_SWHASH_HASH(aobj, pageidx);
                    231:        page_tag = UAO_SWHASH_ELT_TAG(pageidx);
1.1       mrg       232:
1.5       mrg       233:        /*
                    234:         * now search the bucket for the requested tag
                    235:         */
1.39.2.3  nathanw   236:
1.37      chs       237:        LIST_FOREACH(elt, swhash, list) {
1.39.2.3  nathanw   238:                if (elt->tag == page_tag) {
                    239:                        return elt;
                    240:                }
1.5       mrg       241:        }
1.39.2.3  nathanw   242:        if (!create) {
1.5       mrg       243:                return NULL;
1.39.2.3  nathanw   244:        }
1.5       mrg       245:
                    246:        /*
1.12      thorpej   247:         * allocate a new entry for the bucket and init/insert it in
1.5       mrg       248:         */
1.39.2.3  nathanw   249:
                    250:        elt = pool_get(&uao_swhash_elt_pool, PR_NOWAIT);
                    251:        if (elt == NULL) {
                    252:                return NULL;
                    253:        }
1.5       mrg       254:        LIST_INSERT_HEAD(swhash, elt, list);
                    255:        elt->tag = page_tag;
                    256:        elt->count = 0;
1.9       perry     257:        memset(elt->slots, 0, sizeof(elt->slots));
1.39.2.3  nathanw   258:        return elt;
1.1       mrg       259: }
                    260:
                    261: /*
                    262:  * uao_find_swslot: find the swap slot number for an aobj/pageidx
                    263:  *
1.39.2.2  nathanw   264:  * => object must be locked by caller
1.1       mrg       265:  */
1.39.2.4! nathanw   266:
        !           267: int
        !           268: uao_find_swslot(uobj, pageidx)
        !           269:        struct uvm_object *uobj;
1.11      drochner  270:        int pageidx;
1.1       mrg       271: {
1.39.2.4! nathanw   272:        struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
        !           273:        struct uao_swhash_elt *elt;
1.1       mrg       274:
1.5       mrg       275:        /*
                    276:         * if noswap flag is set, then we never return a slot
                    277:         */
1.1       mrg       278:
1.5       mrg       279:        if (aobj->u_flags & UAO_FLAG_NOSWAP)
                    280:                return(0);
1.1       mrg       281:
1.5       mrg       282:        /*
                    283:         * if hashing, look in hash table.
                    284:         */
1.1       mrg       285:
1.5       mrg       286:        if (UAO_USES_SWHASH(aobj)) {
1.39.2.4! nathanw   287:                elt = uao_find_swhash_elt(aobj, pageidx, FALSE);
1.5       mrg       288:                if (elt)
                    289:                        return(UAO_SWHASH_ELT_PAGESLOT(elt, pageidx));
                    290:                else
1.31      thorpej   291:                        return(0);
1.5       mrg       292:        }
1.1       mrg       293:
1.39.2.2  nathanw   294:        /*
1.5       mrg       295:         * otherwise, look in the array
                    296:         */
1.39.2.4! nathanw   297:
1.5       mrg       298:        return(aobj->u_swslots[pageidx]);
1.1       mrg       299: }
                    300:
                    301: /*
                    302:  * uao_set_swslot: set the swap slot for a page in an aobj.
                    303:  *
                    304:  * => setting a slot to zero frees the slot
                    305:  * => object must be locked by caller
1.39.2.3  nathanw   306:  * => we return the old slot number, or -1 if we failed to allocate
                    307:  *    memory to record the new slot number
1.1       mrg       308:  */
1.39.2.4! nathanw   309:
1.5       mrg       310: int
                    311: uao_set_swslot(uobj, pageidx, slot)
                    312:        struct uvm_object *uobj;
                    313:        int pageidx, slot;
                    314: {
                    315:        struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.39.2.3  nathanw   316:        struct uao_swhash_elt *elt;
1.5       mrg       317:        int oldslot;
                    318:        UVMHIST_FUNC("uao_set_swslot"); UVMHIST_CALLED(pdhist);
                    319:        UVMHIST_LOG(pdhist, "aobj %p pageidx %d slot %d",
                    320:            aobj, pageidx, slot, 0);
1.1       mrg       321:
1.5       mrg       322:        /*
1.39.2.4! nathanw   323:         * if noswap flag is set, then we can't set a non-zero slot.
1.5       mrg       324:         */
1.1       mrg       325:
1.5       mrg       326:        if (aobj->u_flags & UAO_FLAG_NOSWAP) {
                    327:                if (slot == 0)
1.39.2.4! nathanw   328:                        return(0);
1.1       mrg       329:
1.5       mrg       330:                printf("uao_set_swslot: uobj = %p\n", uobj);
1.39.2.4! nathanw   331:                panic("uao_set_swslot: NOSWAP object");
1.5       mrg       332:        }
1.1       mrg       333:
1.5       mrg       334:        /*
                    335:         * are we using a hash table?  if so, add it in the hash.
                    336:         */
1.1       mrg       337:
1.5       mrg       338:        if (UAO_USES_SWHASH(aobj)) {
1.39      chs       339:
1.12      thorpej   340:                /*
                    341:                 * Avoid allocating an entry just to free it again if
                    342:                 * the page had not swap slot in the first place, and
                    343:                 * we are freeing.
                    344:                 */
1.39      chs       345:
1.39.2.4! nathanw   346:                elt = uao_find_swhash_elt(aobj, pageidx, slot != 0);
1.12      thorpej   347:                if (elt == NULL) {
1.39.2.3  nathanw   348:                        return slot ? -1 : 0;
1.12      thorpej   349:                }
1.5       mrg       350:
                    351:                oldslot = UAO_SWHASH_ELT_PAGESLOT(elt, pageidx);
                    352:                UAO_SWHASH_ELT_PAGESLOT(elt, pageidx) = slot;
                    353:
                    354:                /*
                    355:                 * now adjust the elt's reference counter and free it if we've
                    356:                 * dropped it to zero.
                    357:                 */
                    358:
                    359:                if (slot) {
                    360:                        if (oldslot == 0)
                    361:                                elt->count++;
1.39.2.3  nathanw   362:                } else {
                    363:                        if (oldslot)
1.5       mrg       364:                                elt->count--;
                    365:
                    366:                        if (elt->count == 0) {
                    367:                                LIST_REMOVE(elt, list);
1.12      thorpej   368:                                pool_put(&uao_swhash_elt_pool, elt);
1.5       mrg       369:                        }
                    370:                }
1.39.2.2  nathanw   371:        } else {
1.5       mrg       372:                /* we are using an array */
                    373:                oldslot = aobj->u_swslots[pageidx];
                    374:                aobj->u_swslots[pageidx] = slot;
                    375:        }
                    376:        return (oldslot);
1.1       mrg       377: }
                    378:
                    379: /*
                    380:  * end of hash/array functions
                    381:  */
                    382:
                    383: /*
                    384:  * uao_free: free all resources held by an aobj, and then free the aobj
                    385:  *
                    386:  * => the aobj should be dead
                    387:  */
1.39.2.4! nathanw   388:
1.1       mrg       389: static void
                    390: uao_free(aobj)
1.5       mrg       391:        struct uvm_aobj *aobj;
1.1       mrg       392: {
1.39.2.4! nathanw   393:        int swpgonlydelta = 0;
1.1       mrg       394:
1.27      chs       395:        simple_unlock(&aobj->u_obj.vmobjlock);
1.5       mrg       396:        if (UAO_USES_SWHASH(aobj)) {
                    397:                int i, hashbuckets = aobj->u_swhashmask + 1;
1.1       mrg       398:
1.5       mrg       399:                /*
                    400:                 * free the swslots from each hash bucket,
                    401:                 * then the hash bucket, and finally the hash table itself.
                    402:                 */
1.39.2.4! nathanw   403:
1.5       mrg       404:                for (i = 0; i < hashbuckets; i++) {
                    405:                        struct uao_swhash_elt *elt, *next;
                    406:
1.27      chs       407:                        for (elt = LIST_FIRST(&aobj->u_swhash[i]);
                    408:                             elt != NULL;
                    409:                             elt = next) {
1.5       mrg       410:                                int j;
                    411:
1.27      chs       412:                                for (j = 0; j < UAO_SWHASH_CLUSTER_SIZE; j++) {
1.5       mrg       413:                                        int slot = elt->slots[j];
                    414:
1.37      chs       415:                                        if (slot == 0) {
                    416:                                                continue;
                    417:                                        }
                    418:                                        uvm_swap_free(slot, 1);
1.39.2.4! nathanw   419:                                        swpgonlydelta++;
1.5       mrg       420:                                }
                    421:
1.27      chs       422:                                next = LIST_NEXT(elt, list);
1.12      thorpej   423:                                pool_put(&uao_swhash_elt_pool, elt);
1.5       mrg       424:                        }
                    425:                }
1.34      thorpej   426:                free(aobj->u_swhash, M_UVMAOBJ);
1.5       mrg       427:        } else {
                    428:                int i;
                    429:
                    430:                /*
                    431:                 * free the array
                    432:                 */
                    433:
1.27      chs       434:                for (i = 0; i < aobj->u_pages; i++) {
1.5       mrg       435:                        int slot = aobj->u_swslots[i];
                    436:
1.18      chs       437:                        if (slot) {
1.5       mrg       438:                                uvm_swap_free(slot, 1);
1.39.2.4! nathanw   439:                                swpgonlydelta++;
1.18      chs       440:                        }
1.5       mrg       441:                }
1.34      thorpej   442:                free(aobj->u_swslots, M_UVMAOBJ);
1.1       mrg       443:        }
                    444:
1.5       mrg       445:        /*
                    446:         * finally free the aobj itself
                    447:         */
1.39.2.4! nathanw   448:
1.12      thorpej   449:        pool_put(&uvm_aobj_pool, aobj);
1.39.2.4! nathanw   450:
        !           451:        /*
        !           452:         * adjust the counter of pages only in swap for all
        !           453:         * the swap slots we've freed.
        !           454:         */
        !           455:
        !           456:        simple_lock(&uvm.swap_data_lock);
        !           457:        uvmexp.swpgonly -= swpgonlydelta;
        !           458:        simple_unlock(&uvm.swap_data_lock);
1.1       mrg       459: }
                    460:
                    461: /*
                    462:  * pager functions
                    463:  */
                    464:
                    465: /*
                    466:  * uao_create: create an aobj of the given size and return its uvm_object.
                    467:  *
                    468:  * => for normal use, flags are always zero
                    469:  * => for the kernel object, the flags are:
                    470:  *     UAO_FLAG_KERNOBJ - allocate the kernel object (can only happen once)
                    471:  *     UAO_FLAG_KERNSWAP - enable swapping of kernel object ("           ")
                    472:  */
1.39.2.4! nathanw   473:
1.5       mrg       474: struct uvm_object *
                    475: uao_create(size, flags)
1.10      eeh       476:        vsize_t size;
1.5       mrg       477:        int flags;
                    478: {
1.39.2.4! nathanw   479:        static struct uvm_aobj kernel_object_store;
        !           480:        static int kobj_alloced = 0;
1.15      chs       481:        int pages = round_page(size) >> PAGE_SHIFT;
1.5       mrg       482:        struct uvm_aobj *aobj;
1.1       mrg       483:
1.5       mrg       484:        /*
1.27      chs       485:         * malloc a new aobj unless we are asked for the kernel object
                    486:         */
1.5       mrg       487:
1.39.2.4! nathanw   488:        if (flags & UAO_FLAG_KERNOBJ) {
        !           489:                KASSERT(!kobj_alloced);
1.5       mrg       490:                aobj = &kernel_object_store;
                    491:                aobj->u_pages = pages;
1.39.2.4! nathanw   492:                aobj->u_flags = UAO_FLAG_NOSWAP;
1.5       mrg       493:                aobj->u_obj.uo_refs = UVM_OBJ_KERN;
                    494:                kobj_alloced = UAO_FLAG_KERNOBJ;
                    495:        } else if (flags & UAO_FLAG_KERNSWAP) {
1.39.2.4! nathanw   496:                KASSERT(kobj_alloced == UAO_FLAG_KERNOBJ);
1.5       mrg       497:                aobj = &kernel_object_store;
                    498:                kobj_alloced = UAO_FLAG_KERNSWAP;
1.39.2.4! nathanw   499:        } else {
1.12      thorpej   500:                aobj = pool_get(&uvm_aobj_pool, PR_WAITOK);
1.5       mrg       501:                aobj->u_pages = pages;
1.39.2.4! nathanw   502:                aobj->u_flags = 0;
        !           503:                aobj->u_obj.uo_refs = 1;
1.5       mrg       504:        }
1.1       mrg       505:
1.5       mrg       506:        /*
                    507:         * allocate hash/array if necessary
                    508:         *
                    509:         * note: in the KERNSWAP case no need to worry about locking since
                    510:         * we are still booting we should be the only thread around.
                    511:         */
1.39.2.4! nathanw   512:
1.5       mrg       513:        if (flags == 0 || (flags & UAO_FLAG_KERNSWAP) != 0) {
                    514:                int mflags = (flags & UAO_FLAG_KERNSWAP) != 0 ?
                    515:                    M_NOWAIT : M_WAITOK;
                    516:
                    517:                /* allocate hash table or array depending on object size */
1.27      chs       518:                if (UAO_USES_SWHASH(aobj)) {
1.5       mrg       519:                        aobj->u_swhash = hashinit(UAO_SWHASH_BUCKETS(aobj),
1.35      ad        520:                            HASH_LIST, M_UVMAOBJ, mflags, &aobj->u_swhashmask);
1.5       mrg       521:                        if (aobj->u_swhash == NULL)
                    522:                                panic("uao_create: hashinit swhash failed");
                    523:                } else {
1.34      thorpej   524:                        aobj->u_swslots = malloc(pages * sizeof(int),
1.5       mrg       525:                            M_UVMAOBJ, mflags);
                    526:                        if (aobj->u_swslots == NULL)
                    527:                                panic("uao_create: malloc swslots failed");
1.9       perry     528:                        memset(aobj->u_swslots, 0, pages * sizeof(int));
1.5       mrg       529:                }
                    530:
                    531:                if (flags) {
                    532:                        aobj->u_flags &= ~UAO_FLAG_NOSWAP; /* clear noswap */
                    533:                        return(&aobj->u_obj);
                    534:                }
                    535:        }
                    536:
                    537:        /*
                    538:         * init aobj fields
                    539:         */
1.39.2.4! nathanw   540:
1.5       mrg       541:        simple_lock_init(&aobj->u_obj.vmobjlock);
                    542:        aobj->u_obj.pgops = &aobj_pager;
                    543:        TAILQ_INIT(&aobj->u_obj.memq);
                    544:        aobj->u_obj.uo_npages = 0;
1.1       mrg       545:
1.5       mrg       546:        /*
                    547:         * now that aobj is ready, add it to the global list
                    548:         */
1.39.2.4! nathanw   549:
1.5       mrg       550:        simple_lock(&uao_list_lock);
                    551:        LIST_INSERT_HEAD(&uao_list, aobj, u_list);
                    552:        simple_unlock(&uao_list_lock);
                    553:        return(&aobj->u_obj);
1.1       mrg       554: }
                    555:
                    556:
                    557:
                    558: /*
                    559:  * uao_init: set up aobj pager subsystem
                    560:  *
                    561:  * => called at boot time from uvm_pager_init()
                    562:  */
1.39.2.4! nathanw   563:
1.27      chs       564: void
1.39.2.4! nathanw   565: uao_init(void)
1.5       mrg       566: {
1.12      thorpej   567:        static int uao_initialized;
                    568:
                    569:        if (uao_initialized)
                    570:                return;
                    571:        uao_initialized = TRUE;
1.5       mrg       572:        LIST_INIT(&uao_list);
                    573:        simple_lock_init(&uao_list_lock);
1.12      thorpej   574:
1.14      thorpej   575:        /*
                    576:         * NOTE: Pages fror this pool must not come from a pageable
                    577:         * kernel map!
                    578:         */
1.39.2.4! nathanw   579:
1.12      thorpej   580:        pool_init(&uao_swhash_elt_pool, sizeof(struct uao_swhash_elt),
1.13      thorpej   581:            0, 0, 0, "uaoeltpl", 0, NULL, NULL, M_UVMAOBJ);
1.12      thorpej   582:        pool_init(&uvm_aobj_pool, sizeof(struct uvm_aobj), 0, 0, 0,
                    583:            "aobjpl", 0,
                    584:            pool_page_alloc_nointr, pool_page_free_nointr, M_UVMAOBJ);
1.1       mrg       585: }
                    586:
                    587: /*
                    588:  * uao_reference: add a ref to an aobj
                    589:  *
1.27      chs       590:  * => aobj must be unlocked
                    591:  * => just lock it and call the locked version
1.1       mrg       592:  */
1.39.2.4! nathanw   593:
1.5       mrg       594: void
                    595: uao_reference(uobj)
                    596:        struct uvm_object *uobj;
1.1       mrg       597: {
1.27      chs       598:        simple_lock(&uobj->vmobjlock);
                    599:        uao_reference_locked(uobj);
                    600:        simple_unlock(&uobj->vmobjlock);
                    601: }
                    602:
                    603: /*
                    604:  * uao_reference_locked: add a ref to an aobj that is already locked
                    605:  *
                    606:  * => aobj must be locked
                    607:  * this needs to be separate from the normal routine
                    608:  * since sometimes we need to add a reference to an aobj when
                    609:  * it's already locked.
                    610:  */
1.39.2.4! nathanw   611:
1.27      chs       612: void
                    613: uao_reference_locked(uobj)
                    614:        struct uvm_object *uobj;
                    615: {
1.5       mrg       616:        UVMHIST_FUNC("uao_reference"); UVMHIST_CALLED(maphist);
1.1       mrg       617:
1.5       mrg       618:        /*
                    619:         * kernel_object already has plenty of references, leave it alone.
                    620:         */
1.1       mrg       621:
1.20      thorpej   622:        if (UVM_OBJ_IS_KERN_OBJECT(uobj))
1.5       mrg       623:                return;
1.1       mrg       624:
1.39.2.4! nathanw   625:        uobj->uo_refs++;
1.39.2.2  nathanw   626:        UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)",
1.27      chs       627:                    uobj, uobj->uo_refs,0,0);
1.1       mrg       628: }
                    629:
                    630: /*
                    631:  * uao_detach: drop a reference to an aobj
                    632:  *
1.27      chs       633:  * => aobj must be unlocked
                    634:  * => just lock it and call the locked version
1.1       mrg       635:  */
1.39.2.4! nathanw   636:
1.5       mrg       637: void
                    638: uao_detach(uobj)
                    639:        struct uvm_object *uobj;
                    640: {
1.27      chs       641:        simple_lock(&uobj->vmobjlock);
                    642:        uao_detach_locked(uobj);
                    643: }
                    644:
                    645: /*
                    646:  * uao_detach_locked: drop a reference to an aobj
                    647:  *
                    648:  * => aobj must be locked, and is unlocked (or freed) upon return.
                    649:  * this needs to be separate from the normal routine
                    650:  * since sometimes we need to detach from an aobj when
                    651:  * it's already locked.
                    652:  */
1.39.2.4! nathanw   653:
1.27      chs       654: void
                    655: uao_detach_locked(uobj)
                    656:        struct uvm_object *uobj;
                    657: {
1.5       mrg       658:        struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.39.2.4! nathanw   659:        struct vm_page *pg;
1.5       mrg       660:        UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist);
1.1       mrg       661:
1.5       mrg       662:        /*
                    663:         * detaching from kernel_object is a noop.
                    664:         */
1.39.2.4! nathanw   665:
1.27      chs       666:        if (UVM_OBJ_IS_KERN_OBJECT(uobj)) {
                    667:                simple_unlock(&uobj->vmobjlock);
1.5       mrg       668:                return;
1.27      chs       669:        }
1.5       mrg       670:
                    671:        UVMHIST_LOG(maphist,"  (uobj=0x%x)  ref=%d", uobj,uobj->uo_refs,0,0);
1.39.2.4! nathanw   672:        uobj->uo_refs--;
        !           673:        if (uobj->uo_refs) {
1.5       mrg       674:                simple_unlock(&uobj->vmobjlock);
                    675:                UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
                    676:                return;
                    677:        }
                    678:
                    679:        /*
                    680:         * remove the aobj from the global list.
                    681:         */
1.39.2.4! nathanw   682:
1.5       mrg       683:        simple_lock(&uao_list_lock);
                    684:        LIST_REMOVE(aobj, u_list);
                    685:        simple_unlock(&uao_list_lock);
                    686:
                    687:        /*
1.39.2.4! nathanw   688:         * free all the pages left in the aobj.  for each page,
        !           689:         * when the page is no longer busy (and thus after any disk i/o that
        !           690:         * it's involved in is complete), release any swap resources and
        !           691:         * free the page itself.
1.5       mrg       692:         */
1.39.2.4! nathanw   693:
        !           694:        uvm_lock_pageq();
        !           695:        while ((pg = TAILQ_FIRST(&uobj->memq)) != NULL) {
        !           696:                pmap_page_protect(pg, VM_PROT_NONE);
1.5       mrg       697:                if (pg->flags & PG_BUSY) {
1.39.2.4! nathanw   698:                        pg->flags |= PG_WANTED;
        !           699:                        uvm_unlock_pageq();
        !           700:                        UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, FALSE,
        !           701:                            "uao_det", 0);
        !           702:                        simple_lock(&uobj->vmobjlock);
        !           703:                        uvm_lock_pageq();
1.5       mrg       704:                        continue;
                    705:                }
1.18      chs       706:                uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
1.5       mrg       707:                uvm_pagefree(pg);
                    708:        }
1.39.2.4! nathanw   709:        uvm_unlock_pageq();
1.1       mrg       710:
1.5       mrg       711:        /*
1.39.2.4! nathanw   712:         * finally, free the aobj itself.
1.5       mrg       713:         */
1.1       mrg       714:
1.5       mrg       715:        uao_free(aobj);
                    716: }
1.1       mrg       717:
                    718: /*
1.39.2.4! nathanw   719:  * uao_put: flush pages out of a uvm object
1.22      thorpej   720:  *
                    721:  * => object should be locked by caller.  we may _unlock_ the object
                    722:  *     if (and only if) we need to clean a page (PGO_CLEANIT).
                    723:  *     XXXJRT Currently, however, we don't.  In the case of cleaning
                    724:  *     XXXJRT a page, we simply just deactivate it.  Should probably
                    725:  *     XXXJRT handle this better, in the future (although "flushing"
                    726:  *     XXXJRT anonymous memory isn't terribly important).
                    727:  * => if PGO_CLEANIT is not set, then we will neither unlock the object
                    728:  *     or block.
                    729:  * => if PGO_ALLPAGE is set, then all pages in the object are valid targets
                    730:  *     for flushing.
                    731:  * => NOTE: we rely on the fact that the object's memq is a TAILQ and
                    732:  *     that new pages are inserted on the tail end of the list.  thus,
                    733:  *     we can make a complete pass through the object in one go by starting
                    734:  *     at the head and working towards the tail (new pages are put in
                    735:  *     front of us).
                    736:  * => NOTE: we are allowed to lock the page queues, so the caller
                    737:  *     must not be holding the lock on them [e.g. pagedaemon had
                    738:  *     better not call us with the queues locked]
                    739:  * => we return TRUE unless we encountered some sort of I/O error
                    740:  *     XXXJRT currently never happens, as we never directly initiate
                    741:  *     XXXJRT I/O
                    742:  *
                    743:  * note on page traversal:
                    744:  *     we can traverse the pages in an object either by going down the
                    745:  *     linked list in "uobj->memq", or we can go over the address range
                    746:  *     by page doing hash table lookups for each address.  depending
                    747:  *     on how many pages are in the object it may be cheaper to do one
                    748:  *     or the other.  we set "by_list" to true if we are using memq.
                    749:  *     if the cost of a hash lookup was equal to the cost of the list
                    750:  *     traversal we could compare the number of pages in the start->stop
                    751:  *     range to the total number of pages in the object.  however, it
                    752:  *     seems that a hash table lookup is more expensive than the linked
                    753:  *     list traversal, so we multiply the number of pages in the
                    754:  *     start->stop range by a penalty which we define below.
1.1       mrg       755:  */
1.22      thorpej   756:
1.39.2.4! nathanw   757: int
        !           758: uao_put(uobj, start, stop, flags)
1.5       mrg       759:        struct uvm_object *uobj;
1.28      kleink    760:        voff_t start, stop;
1.5       mrg       761:        int flags;
                    762: {
1.39.2.4! nathanw   763:        struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
        !           764:        struct vm_page *pg, *nextpg;
        !           765:        boolean_t by_list;
1.28      kleink    766:        voff_t curoff;
1.39.2.4! nathanw   767:        UVMHIST_FUNC("uao_put"); UVMHIST_CALLED(maphist);
1.22      thorpej   768:
1.39.2.4! nathanw   769:        curoff = 0;
1.22      thorpej   770:        if (flags & PGO_ALLPAGES) {
                    771:                start = 0;
                    772:                stop = aobj->u_pages << PAGE_SHIFT;
                    773:                by_list = TRUE;         /* always go by the list */
                    774:        } else {
                    775:                start = trunc_page(start);
                    776:                stop = round_page(stop);
                    777:                if (stop > (aobj->u_pages << PAGE_SHIFT)) {
                    778:                        printf("uao_flush: strange, got an out of range "
                    779:                            "flush (fixed)\n");
                    780:                        stop = aobj->u_pages << PAGE_SHIFT;
                    781:                }
                    782:                by_list = (uobj->uo_npages <=
1.39.2.4! nathanw   783:                    ((stop - start) >> PAGE_SHIFT) * UVM_PAGE_HASH_PENALTY);
1.22      thorpej   784:        }
                    785:        UVMHIST_LOG(maphist,
                    786:            " flush start=0x%lx, stop=0x%x, by_list=%d, flags=0x%x",
                    787:            start, stop, by_list, flags);
1.1       mrg       788:
1.5       mrg       789:        /*
1.22      thorpej   790:         * Don't need to do any work here if we're not freeing
                    791:         * or deactivating pages.
                    792:         */
1.39.2.4! nathanw   793:
1.22      thorpej   794:        if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
1.39.2.4! nathanw   795:                simple_unlock(&uobj->vmobjlock);
        !           796:                return 0;
1.22      thorpej   797:        }
                    798:
1.5       mrg       799:        /*
1.39.2.4! nathanw   800:         * now do it.  note: we must update nextpg in the body of loop or we
        !           801:         * will get stuck.  we need to use nextpg because we may free "pg"
1.22      thorpej   802:         * before doing the next loop.
1.21      thorpej   803:         */
1.22      thorpej   804:
                    805:        if (by_list) {
1.39.2.4! nathanw   806:                pg = TAILQ_FIRST(&uobj->memq);
1.22      thorpej   807:        } else {
                    808:                curoff = start;
1.39.2.4! nathanw   809:                pg = uvm_pagelookup(uobj, curoff);
1.22      thorpej   810:        }
                    811:
1.39.2.4! nathanw   812:        nextpg = NULL;
        !           813:        uvm_lock_pageq();
1.22      thorpej   814:
                    815:        /* locked: both page queues and uobj */
1.39.2.4! nathanw   816:        for ( ; (by_list && pg != NULL) ||
        !           817:            (!by_list && curoff < stop) ; pg = nextpg) {
1.22      thorpej   818:                if (by_list) {
1.39.2.4! nathanw   819:                        nextpg = TAILQ_NEXT(pg, listq);
        !           820:                        if (pg->offset < start || pg->offset >= stop)
1.22      thorpej   821:                                continue;
                    822:                } else {
                    823:                        curoff += PAGE_SIZE;
                    824:                        if (curoff < stop)
1.39.2.4! nathanw   825:                                nextpg = uvm_pagelookup(uobj, curoff);
        !           826:                        if (pg == NULL)
1.22      thorpej   827:                                continue;
                    828:                }
                    829:                switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
1.39.2.4! nathanw   830:
1.22      thorpej   831:                /*
                    832:                 * XXX In these first 3 cases, we always just
                    833:                 * XXX deactivate the page.  We may want to
                    834:                 * XXX handle the different cases more specifically
                    835:                 * XXX in the future.
                    836:                 */
1.39.2.4! nathanw   837:
1.22      thorpej   838:                case PGO_CLEANIT|PGO_FREE:
                    839:                case PGO_CLEANIT|PGO_DEACTIVATE:
                    840:                case PGO_DEACTIVATE:
1.25      thorpej   841:  deactivate_it:
1.22      thorpej   842:                        /* skip the page if it's loaned or wired */
1.39.2.4! nathanw   843:                        if (pg->loan_count != 0 || pg->wire_count != 0)
1.22      thorpej   844:                                continue;
                    845:
                    846:                        /* ...and deactivate the page. */
1.39.2.4! nathanw   847:                        pmap_clear_reference(pg);
        !           848:                        uvm_pagedeactivate(pg);
1.22      thorpej   849:                        continue;
                    850:
                    851:                case PGO_FREE:
1.39.2.4! nathanw   852:
1.25      thorpej   853:                        /*
                    854:                         * If there are multiple references to
                    855:                         * the object, just deactivate the page.
                    856:                         */
1.39.2.4! nathanw   857:
1.25      thorpej   858:                        if (uobj->uo_refs > 1)
                    859:                                goto deactivate_it;
                    860:
1.22      thorpej   861:                        /* XXX skip the page if it's loaned or wired */
1.39.2.4! nathanw   862:                        if (pg->loan_count != 0 || pg->wire_count != 0)
1.22      thorpej   863:                                continue;
                    864:
                    865:                        /*
1.39.2.4! nathanw   866:                         * wait if the page is busy, then free the swap slot
        !           867:                         * and the page.
1.22      thorpej   868:                         */
                    869:
1.39.2.4! nathanw   870:                        pmap_page_protect(pg, VM_PROT_NONE);
        !           871:                        while (pg->flags & PG_BUSY) {
        !           872:                                pg->flags |= PG_WANTED;
        !           873:                                uvm_unlock_pageq();
        !           874:                                UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, 0,
        !           875:                                    "uao_put", 0);
        !           876:                                simple_lock(&uobj->vmobjlock);
        !           877:                                uvm_lock_pageq();
        !           878:                        }
        !           879:                        uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
        !           880:                        uvm_pagefree(pg);
1.22      thorpej   881:                        continue;
                    882:                }
                    883:        }
                    884:        uvm_unlock_pageq();
1.39.2.4! nathanw   885:        simple_unlock(&uobj->vmobjlock);
        !           886:        return 0;
1.1       mrg       887: }
                    888:
                    889: /*
                    890:  * uao_get: fetch me a page
                    891:  *
                    892:  * we have three cases:
                    893:  * 1: page is resident     -> just return the page.
                    894:  * 2: page is zero-fill    -> allocate a new page and zero it.
                    895:  * 3: page is swapped out  -> fetch the page from swap.
                    896:  *
                    897:  * cases 1 and 2 can be handled with PGO_LOCKED, case 3 cannot.
                    898:  * so, if the "center" page hits case 3 (or any page, with PGO_ALLPAGES),
1.39.2.1  nathanw   899:  * then we will need to return EBUSY.
1.1       mrg       900:  *
                    901:  * => prefer map unlocked (not required)
                    902:  * => object must be locked!  we will _unlock_ it before starting any I/O.
                    903:  * => flags: PGO_ALLPAGES: get all of the pages
                    904:  *           PGO_LOCKED: fault data structures are locked
                    905:  * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
                    906:  * => NOTE: caller must check for released pages!!
                    907:  */
1.39.2.4! nathanw   908:
1.5       mrg       909: static int
                    910: uao_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags)
                    911:        struct uvm_object *uobj;
1.28      kleink    912:        voff_t offset;
1.5       mrg       913:        struct vm_page **pps;
                    914:        int *npagesp;
                    915:        int centeridx, advice, flags;
                    916:        vm_prot_t access_type;
                    917: {
                    918:        struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.28      kleink    919:        voff_t current_offset;
1.39.2.2  nathanw   920:        struct vm_page *ptmp;
1.39.2.4! nathanw   921:        int lcv, gotpages, maxpages, swslot, error, pageidx;
1.5       mrg       922:        boolean_t done;
                    923:        UVMHIST_FUNC("uao_get"); UVMHIST_CALLED(pdhist);
                    924:
1.27      chs       925:        UVMHIST_LOG(pdhist, "aobj=%p offset=%d, flags=%d",
                    926:                    aobj, offset, flags,0);
1.37      chs       927:
1.5       mrg       928:        /*
                    929:         * get number of pages
                    930:         */
1.39.2.4! nathanw   931:
1.5       mrg       932:        maxpages = *npagesp;
                    933:
                    934:        /*
                    935:         * step 1: handled the case where fault data structures are locked.
                    936:         */
1.1       mrg       937:
1.5       mrg       938:        if (flags & PGO_LOCKED) {
1.39.2.4! nathanw   939:
1.5       mrg       940:                /*
                    941:                 * step 1a: get pages that are already resident.   only do
                    942:                 * this if the data structures are locked (i.e. the first
                    943:                 * time through).
                    944:                 */
                    945:
                    946:                done = TRUE;    /* be optimistic */
                    947:                gotpages = 0;   /* # of pages we got so far */
                    948:                for (lcv = 0, current_offset = offset ; lcv < maxpages ;
                    949:                    lcv++, current_offset += PAGE_SIZE) {
                    950:                        /* do we care about this page?  if not, skip it */
                    951:                        if (pps[lcv] == PGO_DONTCARE)
                    952:                                continue;
                    953:                        ptmp = uvm_pagelookup(uobj, current_offset);
                    954:
                    955:                        /*
1.30      thorpej   956:                         * if page is new, attempt to allocate the page,
                    957:                         * zero-fill'd.
1.5       mrg       958:                         */
1.39.2.4! nathanw   959:
        !           960:                        if (ptmp == NULL && uao_find_swslot(&aobj->u_obj,
1.15      chs       961:                            current_offset >> PAGE_SHIFT) == 0) {
1.5       mrg       962:                                ptmp = uvm_pagealloc(uobj, current_offset,
1.30      thorpej   963:                                    NULL, UVM_PGA_ZERO);
1.5       mrg       964:                                if (ptmp) {
                    965:                                        /* new page */
                    966:                                        ptmp->flags &= ~(PG_BUSY|PG_FAKE);
                    967:                                        ptmp->pqflags |= PQ_AOBJ;
                    968:                                        UVM_PAGE_OWN(ptmp, NULL);
                    969:                                }
                    970:                        }
                    971:
                    972:                        /*
1.39.2.4! nathanw   973:                         * to be useful must get a non-busy page
1.5       mrg       974:                         */
1.39.2.4! nathanw   975:
        !           976:                        if (ptmp == NULL || (ptmp->flags & PG_BUSY) != 0) {
1.5       mrg       977:                                if (lcv == centeridx ||
                    978:                                    (flags & PGO_ALLPAGES) != 0)
                    979:                                        /* need to do a wait or I/O! */
1.39.2.2  nathanw   980:                                        done = FALSE;
1.5       mrg       981:                                        continue;
                    982:                        }
                    983:
                    984:                        /*
                    985:                         * useful page: busy/lock it and plug it in our
                    986:                         * result array
                    987:                         */
1.39.2.4! nathanw   988:
1.5       mrg       989:                        /* caller must un-busy this page */
1.39.2.2  nathanw   990:                        ptmp->flags |= PG_BUSY;
1.5       mrg       991:                        UVM_PAGE_OWN(ptmp, "uao_get1");
                    992:                        pps[lcv] = ptmp;
                    993:                        gotpages++;
1.39.2.4! nathanw   994:                }
1.5       mrg       995:
                    996:                /*
                    997:                 * step 1b: now we've either done everything needed or we
                    998:                 * to unlock and do some waiting or I/O.
                    999:                 */
                   1000:
                   1001:                UVMHIST_LOG(pdhist, "<- done (done=%d)", done, 0,0,0);
                   1002:                *npagesp = gotpages;
                   1003:                if (done)
1.39.2.4! nathanw  1004:                        return 0;
1.5       mrg      1005:                else
1.39.2.4! nathanw  1006:                        return EBUSY;
1.1       mrg      1007:        }
                   1008:
1.5       mrg      1009:        /*
                   1010:         * step 2: get non-resident or busy pages.
                   1011:         * object is locked.   data structures are unlocked.
                   1012:         */
                   1013:
                   1014:        for (lcv = 0, current_offset = offset ; lcv < maxpages ;
                   1015:            lcv++, current_offset += PAGE_SIZE) {
1.27      chs      1016:
1.5       mrg      1017:                /*
                   1018:                 * - skip over pages we've already gotten or don't want
                   1019:                 * - skip over pages we don't _have_ to get
                   1020:                 */
1.27      chs      1021:
1.5       mrg      1022:                if (pps[lcv] != NULL ||
                   1023:                    (lcv != centeridx && (flags & PGO_ALLPAGES) == 0))
                   1024:                        continue;
                   1025:
1.27      chs      1026:                pageidx = current_offset >> PAGE_SHIFT;
                   1027:
1.5       mrg      1028:                /*
                   1029:                 * we have yet to locate the current page (pps[lcv]).   we
                   1030:                 * first look for a page that is already at the current offset.
                   1031:                 * if we find a page, we check to see if it is busy or
                   1032:                 * released.  if that is the case, then we sleep on the page
                   1033:                 * until it is no longer busy or released and repeat the lookup.
                   1034:                 * if the page we found is neither busy nor released, then we
                   1035:                 * busy it (so we own it) and plug it into pps[lcv].   this
                   1036:                 * 'break's the following while loop and indicates we are
                   1037:                 * ready to move on to the next page in the "lcv" loop above.
                   1038:                 *
                   1039:                 * if we exit the while loop with pps[lcv] still set to NULL,
                   1040:                 * then it means that we allocated a new busy/fake/clean page
                   1041:                 * ptmp in the object and we need to do I/O to fill in the data.
                   1042:                 */
                   1043:
                   1044:                /* top of "pps" while loop */
                   1045:                while (pps[lcv] == NULL) {
                   1046:                        /* look for a resident page */
                   1047:                        ptmp = uvm_pagelookup(uobj, current_offset);
                   1048:
                   1049:                        /* not resident?   allocate one now (if we can) */
                   1050:                        if (ptmp == NULL) {
                   1051:
                   1052:                                ptmp = uvm_pagealloc(uobj, current_offset,
1.19      chs      1053:                                    NULL, 0);
1.5       mrg      1054:
                   1055:                                /* out of RAM? */
                   1056:                                if (ptmp == NULL) {
                   1057:                                        simple_unlock(&uobj->vmobjlock);
                   1058:                                        UVMHIST_LOG(pdhist,
                   1059:                                            "sleeping, ptmp == NULL\n",0,0,0,0);
                   1060:                                        uvm_wait("uao_getpage");
                   1061:                                        simple_lock(&uobj->vmobjlock);
1.39.2.2  nathanw  1062:                                        continue;
1.5       mrg      1063:                                }
                   1064:
                   1065:                                /*
                   1066:                                 * safe with PQ's unlocked: because we just
                   1067:                                 * alloc'd the page
                   1068:                                 */
1.39.2.4! nathanw  1069:
1.5       mrg      1070:                                ptmp->pqflags |= PQ_AOBJ;
                   1071:
1.39.2.2  nathanw  1072:                                /*
1.5       mrg      1073:                                 * got new page ready for I/O.  break pps while
                   1074:                                 * loop.  pps[lcv] is still NULL.
                   1075:                                 */
1.39.2.4! nathanw  1076:
1.5       mrg      1077:                                break;
                   1078:                        }
                   1079:
                   1080:                        /* page is there, see if we need to wait on it */
1.39.2.4! nathanw  1081:                        if ((ptmp->flags & PG_BUSY) != 0) {
1.5       mrg      1082:                                ptmp->flags |= PG_WANTED;
                   1083:                                UVMHIST_LOG(pdhist,
                   1084:                                    "sleeping, ptmp->flags 0x%x\n",
                   1085:                                    ptmp->flags,0,0,0);
1.23      thorpej  1086:                                UVM_UNLOCK_AND_WAIT(ptmp, &uobj->vmobjlock,
                   1087:                                    FALSE, "uao_get", 0);
1.5       mrg      1088:                                simple_lock(&uobj->vmobjlock);
1.39.2.4! nathanw  1089:                                continue;
1.5       mrg      1090:                        }
1.39.2.2  nathanw  1091:
                   1092:                        /*
1.5       mrg      1093:                         * if we get here then the page has become resident and
                   1094:                         * unbusy between steps 1 and 2.  we busy it now (so we
                   1095:                         * own it) and set pps[lcv] (so that we exit the while
                   1096:                         * loop).
                   1097:                         */
1.39.2.4! nathanw  1098:
1.5       mrg      1099:                        /* we own it, caller must un-busy */
                   1100:                        ptmp->flags |= PG_BUSY;
                   1101:                        UVM_PAGE_OWN(ptmp, "uao_get2");
                   1102:                        pps[lcv] = ptmp;
                   1103:                }
                   1104:
                   1105:                /*
                   1106:                 * if we own the valid page at the correct offset, pps[lcv] will
                   1107:                 * point to it.   nothing more to do except go to the next page.
                   1108:                 */
1.39.2.4! nathanw  1109:
1.5       mrg      1110:                if (pps[lcv])
                   1111:                        continue;                       /* next lcv */
                   1112:
                   1113:                /*
1.39.2.2  nathanw  1114:                 * we have a "fake/busy/clean" page that we just allocated.
1.5       mrg      1115:                 * do the needed "i/o", either reading from swap or zeroing.
                   1116:                 */
1.39.2.4! nathanw  1117:
        !          1118:                swslot = uao_find_swslot(&aobj->u_obj, pageidx);
1.5       mrg      1119:
                   1120:                /*
                   1121:                 * just zero the page if there's nothing in swap.
                   1122:                 */
1.39.2.4! nathanw  1123:
        !          1124:                if (swslot == 0) {
        !          1125:
1.5       mrg      1126:                        /*
                   1127:                         * page hasn't existed before, just zero it.
                   1128:                         */
1.39.2.4! nathanw  1129:
1.5       mrg      1130:                        uvm_pagezero(ptmp);
1.27      chs      1131:                } else {
1.5       mrg      1132:                        UVMHIST_LOG(pdhist, "pagein from swslot %d",
                   1133:                             swslot, 0,0,0);
                   1134:
                   1135:                        /*
                   1136:                         * page in the swapped-out page.
                   1137:                         * unlock object for i/o, relock when done.
                   1138:                         */
1.39.2.4! nathanw  1139:
1.5       mrg      1140:                        simple_unlock(&uobj->vmobjlock);
1.39.2.4! nathanw  1141:                        error = uvm_swap_get(ptmp, swslot, PGO_SYNCIO);
1.5       mrg      1142:                        simple_lock(&uobj->vmobjlock);
                   1143:
                   1144:                        /*
                   1145:                         * I/O done.  check for errors.
                   1146:                         */
1.39.2.4! nathanw  1147:
        !          1148:                        if (error != 0) {
1.5       mrg      1149:                                UVMHIST_LOG(pdhist, "<- done (error=%d)",
1.39.2.4! nathanw  1150:                                    error,0,0,0);
1.5       mrg      1151:                                if (ptmp->flags & PG_WANTED)
1.24      thorpej  1152:                                        wakeup(ptmp);
1.27      chs      1153:
                   1154:                                /*
                   1155:                                 * remove the swap slot from the aobj
                   1156:                                 * and mark the aobj as having no real slot.
                   1157:                                 * don't free the swap slot, thus preventing
                   1158:                                 * it from being used again.
                   1159:                                 */
1.39.2.4! nathanw  1160:
1.27      chs      1161:                                swslot = uao_set_swslot(&aobj->u_obj, pageidx,
                   1162:                                                        SWSLOT_BAD);
1.39.2.3  nathanw  1163:                                if (swslot != -1) {
                   1164:                                        uvm_swap_markbad(swslot, 1);
                   1165:                                }
1.27      chs      1166:
1.5       mrg      1167:                                uvm_lock_pageq();
                   1168:                                uvm_pagefree(ptmp);
                   1169:                                uvm_unlock_pageq();
                   1170:                                simple_unlock(&uobj->vmobjlock);
1.39.2.4! nathanw  1171:                                return error;
1.5       mrg      1172:                        }
                   1173:                }
                   1174:
1.39.2.2  nathanw  1175:                /*
1.5       mrg      1176:                 * we got the page!   clear the fake flag (indicates valid
                   1177:                 * data now in page) and plug into our result array.   note
1.39.2.2  nathanw  1178:                 * that page is still busy.
1.5       mrg      1179:                 *
                   1180:                 * it is the callers job to:
                   1181:                 * => check if the page is released
                   1182:                 * => unbusy the page
                   1183:                 * => activate the page
                   1184:                 */
                   1185:
1.39.2.4! nathanw  1186:                ptmp->flags &= ~PG_FAKE;
1.5       mrg      1187:                pps[lcv] = ptmp;
1.39.2.4! nathanw  1188:        }
1.1       mrg      1189:
                   1190:        /*
1.5       mrg      1191:         * finally, unlock object and return.
                   1192:         */
1.1       mrg      1193:
                   1194:        simple_unlock(&uobj->vmobjlock);
1.5       mrg      1195:        UVMHIST_LOG(pdhist, "<- done (OK)",0,0,0,0);
1.39.2.4! nathanw  1196:        return 0;
1.1       mrg      1197: }
                   1198:
                   1199: /*
1.18      chs      1200:  * uao_dropswap:  release any swap resources from this aobj page.
1.39.2.2  nathanw  1201:  *
1.18      chs      1202:  * => aobj must be locked or have a reference count of 0.
                   1203:  */
                   1204:
                   1205: void
                   1206: uao_dropswap(uobj, pageidx)
                   1207:        struct uvm_object *uobj;
                   1208:        int pageidx;
                   1209: {
                   1210:        int slot;
                   1211:
                   1212:        slot = uao_set_swslot(uobj, pageidx, 0);
                   1213:        if (slot) {
                   1214:                uvm_swap_free(slot, 1);
                   1215:        }
1.27      chs      1216: }
                   1217:
                   1218:
                   1219: /*
                   1220:  * page in every page in every aobj that is paged-out to a range of swslots.
1.39.2.2  nathanw  1221:  *
1.27      chs      1222:  * => nothing should be locked.
                   1223:  * => returns TRUE if pagein was aborted due to lack of memory.
                   1224:  */
1.39.2.4! nathanw  1225:
1.27      chs      1226: boolean_t
                   1227: uao_swap_off(startslot, endslot)
                   1228:        int startslot, endslot;
                   1229: {
                   1230:        struct uvm_aobj *aobj, *nextaobj;
1.39.2.4! nathanw  1231:        boolean_t rv;
1.27      chs      1232:
                   1233:        /*
                   1234:         * walk the list of all aobjs.
                   1235:         */
                   1236:
                   1237: restart:
                   1238:        simple_lock(&uao_list_lock);
                   1239:        for (aobj = LIST_FIRST(&uao_list);
                   1240:             aobj != NULL;
                   1241:             aobj = nextaobj) {
                   1242:
                   1243:                /*
1.39.2.4! nathanw  1244:                 * try to get the object lock, start all over if we fail.
1.27      chs      1245:                 * most of the time we'll get the aobj lock,
                   1246:                 * so this should be a rare case.
                   1247:                 */
1.39.2.4! nathanw  1248:
1.27      chs      1249:                if (!simple_lock_try(&aobj->u_obj.vmobjlock)) {
                   1250:                        simple_unlock(&uao_list_lock);
                   1251:                        goto restart;
                   1252:                }
                   1253:
                   1254:                /*
                   1255:                 * add a ref to the aobj so it doesn't disappear
                   1256:                 * while we're working.
                   1257:                 */
1.39.2.4! nathanw  1258:
1.27      chs      1259:                uao_reference_locked(&aobj->u_obj);
                   1260:
                   1261:                /*
                   1262:                 * now it's safe to unlock the uao list.
                   1263:                 */
1.39.2.4! nathanw  1264:
1.27      chs      1265:                simple_unlock(&uao_list_lock);
                   1266:
                   1267:                /*
                   1268:                 * page in any pages in the swslot range.
                   1269:                 * if there's an error, abort and return the error.
                   1270:                 */
1.39.2.4! nathanw  1271:
1.27      chs      1272:                rv = uao_pagein(aobj, startslot, endslot);
                   1273:                if (rv) {
                   1274:                        uao_detach_locked(&aobj->u_obj);
                   1275:                        return rv;
                   1276:                }
                   1277:
                   1278:                /*
                   1279:                 * we're done with this aobj.
                   1280:                 * relock the list and drop our ref on the aobj.
                   1281:                 */
1.39.2.4! nathanw  1282:
1.27      chs      1283:                simple_lock(&uao_list_lock);
                   1284:                nextaobj = LIST_NEXT(aobj, u_list);
                   1285:                uao_detach_locked(&aobj->u_obj);
                   1286:        }
                   1287:
                   1288:        /*
                   1289:         * done with traversal, unlock the list
                   1290:         */
                   1291:        simple_unlock(&uao_list_lock);
                   1292:        return FALSE;
                   1293: }
                   1294:
                   1295:
                   1296: /*
                   1297:  * page in any pages from aobj in the given range.
                   1298:  *
                   1299:  * => aobj must be locked and is returned locked.
                   1300:  * => returns TRUE if pagein was aborted due to lack of memory.
                   1301:  */
                   1302: static boolean_t
                   1303: uao_pagein(aobj, startslot, endslot)
                   1304:        struct uvm_aobj *aobj;
                   1305:        int startslot, endslot;
                   1306: {
                   1307:        boolean_t rv;
                   1308:
                   1309:        if (UAO_USES_SWHASH(aobj)) {
                   1310:                struct uao_swhash_elt *elt;
                   1311:                int bucket;
                   1312:
                   1313: restart:
                   1314:                for (bucket = aobj->u_swhashmask; bucket >= 0; bucket--) {
                   1315:                        for (elt = LIST_FIRST(&aobj->u_swhash[bucket]);
                   1316:                             elt != NULL;
                   1317:                             elt = LIST_NEXT(elt, list)) {
                   1318:                                int i;
                   1319:
                   1320:                                for (i = 0; i < UAO_SWHASH_CLUSTER_SIZE; i++) {
                   1321:                                        int slot = elt->slots[i];
                   1322:
                   1323:                                        /*
                   1324:                                         * if the slot isn't in range, skip it.
                   1325:                                         */
1.39.2.4! nathanw  1326:
1.39.2.2  nathanw  1327:                                        if (slot < startslot ||
1.27      chs      1328:                                            slot >= endslot) {
                   1329:                                                continue;
                   1330:                                        }
                   1331:
                   1332:                                        /*
                   1333:                                         * process the page,
                   1334:                                         * the start over on this object
                   1335:                                         * since the swhash elt
                   1336:                                         * may have been freed.
                   1337:                                         */
1.39.2.4! nathanw  1338:
1.27      chs      1339:                                        rv = uao_pagein_page(aobj,
                   1340:                                          UAO_SWHASH_ELT_PAGEIDX_BASE(elt) + i);
                   1341:                                        if (rv) {
                   1342:                                                return rv;
                   1343:                                        }
                   1344:                                        goto restart;
                   1345:                                }
                   1346:                        }
                   1347:                }
                   1348:        } else {
                   1349:                int i;
                   1350:
                   1351:                for (i = 0; i < aobj->u_pages; i++) {
                   1352:                        int slot = aobj->u_swslots[i];
                   1353:
                   1354:                        /*
                   1355:                         * if the slot isn't in range, skip it
                   1356:                         */
1.39.2.4! nathanw  1357:
1.27      chs      1358:                        if (slot < startslot || slot >= endslot) {
                   1359:                                continue;
                   1360:                        }
                   1361:
                   1362:                        /*
                   1363:                         * process the page.
                   1364:                         */
1.39.2.4! nathanw  1365:
1.27      chs      1366:                        rv = uao_pagein_page(aobj, i);
                   1367:                        if (rv) {
                   1368:                                return rv;
                   1369:                        }
                   1370:                }
                   1371:        }
                   1372:
                   1373:        return FALSE;
                   1374: }
                   1375:
                   1376: /*
                   1377:  * page in a page from an aobj.  used for swap_off.
                   1378:  * returns TRUE if pagein was aborted due to lack of memory.
                   1379:  *
                   1380:  * => aobj must be locked and is returned locked.
                   1381:  */
1.39.2.4! nathanw  1382:
1.27      chs      1383: static boolean_t
                   1384: uao_pagein_page(aobj, pageidx)
                   1385:        struct uvm_aobj *aobj;
                   1386:        int pageidx;
                   1387: {
                   1388:        struct vm_page *pg;
                   1389:        int rv, slot, npages;
                   1390:
                   1391:        pg = NULL;
                   1392:        npages = 1;
                   1393:        /* locked: aobj */
                   1394:        rv = uao_get(&aobj->u_obj, pageidx << PAGE_SHIFT,
                   1395:                     &pg, &npages, 0, VM_PROT_READ|VM_PROT_WRITE, 0, 0);
                   1396:        /* unlocked: aobj */
                   1397:
                   1398:        /*
                   1399:         * relock and finish up.
                   1400:         */
                   1401:
1.39.2.4! nathanw  1402:        simple_lock(&aobj->u_obj.vmobjlock);
1.27      chs      1403:        switch (rv) {
1.39.2.1  nathanw  1404:        case 0:
1.27      chs      1405:                break;
                   1406:
1.39.2.1  nathanw  1407:        case EIO:
                   1408:        case ERESTART:
1.39.2.4! nathanw  1409:
1.27      chs      1410:                /*
                   1411:                 * nothing more to do on errors.
1.39.2.1  nathanw  1412:                 * ERESTART can only mean that the anon was freed,
1.27      chs      1413:                 * so again there's nothing to do.
                   1414:                 */
                   1415:
1.39.2.4! nathanw  1416:                return FALSE;
1.27      chs      1417:        }
                   1418:
                   1419:        /*
                   1420:         * ok, we've got the page now.
                   1421:         * mark it as dirty, clear its swslot and un-busy it.
                   1422:         */
1.39.2.4! nathanw  1423:
1.27      chs      1424:        slot = uao_set_swslot(&aobj->u_obj, pageidx, 0);
                   1425:        uvm_swap_free(slot, 1);
                   1426:        pg->flags &= ~(PG_BUSY|PG_CLEAN|PG_FAKE);
                   1427:        UVM_PAGE_OWN(pg, NULL);
                   1428:
                   1429:        /*
1.39.2.4! nathanw  1430:         * deactivate the page (to make sure it's on a page queue).
1.27      chs      1431:         */
1.39.2.4! nathanw  1432:
1.27      chs      1433:        uvm_lock_pageq();
                   1434:        uvm_pagedeactivate(pg);
                   1435:        uvm_unlock_pageq();
                   1436:        return FALSE;
1.1       mrg      1437: }

CVSweb <webmaster@jp.NetBSD.org>