[BACK]Return to pmap.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / arch / powerpc / oea

Annotation of src/sys/arch/powerpc/oea/pmap.c, Revision 1.111

1.111   ! martin      1: /*     $NetBSD: pmap.c,v 1.110 2022/02/18 18:58:15 martin Exp $        */
1.1       matt        2: /*-
                      3:  * Copyright (c) 2001 The NetBSD Foundation, Inc.
                      4:  * All rights reserved.
                      5:  *
                      6:  * This code is derived from software contributed to The NetBSD Foundation
                      7:  * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc.
                      8:  *
1.38      sanjayl     9:  * Support for PPC64 Bridge mode added by Sanjay Lal <sanjayl@kymasys.com>
                     10:  * of Kyma Systems LLC.
                     11:  *
1.1       matt       12:  * Redistribution and use in source and binary forms, with or without
                     13:  * modification, are permitted provided that the following conditions
                     14:  * are met:
                     15:  * 1. Redistributions of source code must retain the above copyright
                     16:  *    notice, this list of conditions and the following disclaimer.
                     17:  * 2. Redistributions in binary form must reproduce the above copyright
                     18:  *    notice, this list of conditions and the following disclaimer in the
                     19:  *    documentation and/or other materials provided with the distribution.
                     20:  *
                     21:  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
                     22:  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
                     23:  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
                     24:  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
                     25:  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
                     26:  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
                     27:  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
                     28:  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
                     29:  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
                     30:  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
                     31:  * POSSIBILITY OF SUCH DAMAGE.
                     32:  */
                     33:
                     34: /*
                     35:  * Copyright (C) 1995, 1996 Wolfgang Solfrank.
                     36:  * Copyright (C) 1995, 1996 TooLs GmbH.
                     37:  * All rights reserved.
                     38:  *
                     39:  * Redistribution and use in source and binary forms, with or without
                     40:  * modification, are permitted provided that the following conditions
                     41:  * are met:
                     42:  * 1. Redistributions of source code must retain the above copyright
                     43:  *    notice, this list of conditions and the following disclaimer.
                     44:  * 2. Redistributions in binary form must reproduce the above copyright
                     45:  *    notice, this list of conditions and the following disclaimer in the
                     46:  *    documentation and/or other materials provided with the distribution.
                     47:  * 3. All advertising materials mentioning features or use of this software
                     48:  *    must display the following acknowledgement:
                     49:  *     This product includes software developed by TooLs GmbH.
                     50:  * 4. The name of TooLs GmbH may not be used to endorse or promote products
                     51:  *    derived from this software without specific prior written permission.
                     52:  *
                     53:  * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
                     54:  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
                     55:  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
                     56:  * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
                     57:  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
                     58:  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
                     59:  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
                     60:  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
                     61:  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
                     62:  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
                     63:  */
1.11      lukem      64:
                     65: #include <sys/cdefs.h>
1.111   ! martin     66: __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.110 2022/02/18 18:58:15 martin Exp $");
1.53      garbled    67:
                     68: #define        PMAP_NOOPNAMES
1.1       matt       69:
1.98      rin        70: #ifdef _KERNEL_OPT
1.1       matt       71: #include "opt_altivec.h"
1.57      matt       72: #include "opt_multiprocessor.h"
1.1       matt       73: #include "opt_pmap.h"
1.98      rin        74: #include "opt_ppcarch.h"
                     75: #endif
1.57      matt       76:
1.1       matt       77: #include <sys/param.h>
                     78: #include <sys/proc.h>
                     79: #include <sys/pool.h>
                     80: #include <sys/queue.h>
                     81: #include <sys/device.h>                /* for evcnt */
                     82: #include <sys/systm.h>
1.50      ad         83: #include <sys/atomic.h>
1.1       matt       84:
                     85: #include <uvm/uvm.h>
1.94      cherry     86: #include <uvm/uvm_physseg.h>
1.1       matt       87:
                     88: #include <machine/powerpc.h>
1.80      matt       89: #include <powerpc/bat.h>
                     90: #include <powerpc/pcb.h>
                     91: #include <powerpc/psl.h>
1.1       matt       92: #include <powerpc/spr.h>
1.71      matt       93: #include <powerpc/oea/spr.h>
                     94: #include <powerpc/oea/sr_601.h>
1.1       matt       95:
                     96: #ifdef ALTIVEC
1.86      matt       97: extern int pmap_use_altivec;
1.1       matt       98: #endif
                     99:
1.21      aymeric   100: #ifdef PMAP_MEMLIMIT
1.53      garbled   101: static paddr_t pmap_memlimit = PMAP_MEMLIMIT;
1.21      aymeric   102: #else
1.53      garbled   103: static paddr_t pmap_memlimit = -PAGE_SIZE;             /* there is no limit */
1.21      aymeric   104: #endif
1.1       matt      105:
1.86      matt      106: extern struct pmap kernel_pmap_;
                    107: static unsigned int pmap_pages_stolen;
                    108: static u_long pmap_pte_valid;
1.1       matt      109: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1.86      matt      110: static u_long pmap_pvo_enter_depth;
                    111: static u_long pmap_pvo_remove_depth;
1.1       matt      112: #endif
                    113:
                    114: #ifndef MSGBUFADDR
                    115: extern paddr_t msgbuf_paddr;
                    116: #endif
                    117:
                    118: static struct mem_region *mem, *avail;
                    119: static u_int mem_cnt, avail_cnt;
                    120:
1.53      garbled   121: #if !defined(PMAP_OEA64) && !defined(PMAP_OEA64_BRIDGE)
                    122: # define       PMAP_OEA 1
                    123: #endif
                    124:
                    125: #if defined(PMAP_OEA)
                    126: #define        _PRIxpte        "lx"
                    127: #else
                    128: #define        _PRIxpte        PRIx64
                    129: #endif
                    130: #define        _PRIxpa         "lx"
                    131: #define        _PRIxva         "lx"
1.54      mlelstv   132: #define        _PRIsr          "lx"
1.53      garbled   133:
1.76      matt      134: #ifdef PMAP_NEEDS_FIXUP
1.53      garbled   135: #if defined(PMAP_OEA)
                    136: #define        PMAPNAME(name)  pmap32_##name
                    137: #elif defined(PMAP_OEA64)
                    138: #define        PMAPNAME(name)  pmap64_##name
                    139: #elif defined(PMAP_OEA64_BRIDGE)
                    140: #define        PMAPNAME(name)  pmap64bridge_##name
                    141: #else
                    142: #error unknown variant for pmap
                    143: #endif
1.76      matt      144: #endif /* PMAP_NEEDS_FIXUP */
1.53      garbled   145:
1.76      matt      146: #ifdef PMAPNAME
1.53      garbled   147: #define        STATIC                  static
                    148: #define pmap_pte_spill         PMAPNAME(pte_spill)
                    149: #define pmap_real_memory       PMAPNAME(real_memory)
                    150: #define pmap_init              PMAPNAME(init)
                    151: #define pmap_virtual_space     PMAPNAME(virtual_space)
                    152: #define pmap_create            PMAPNAME(create)
                    153: #define pmap_reference         PMAPNAME(reference)
                    154: #define pmap_destroy           PMAPNAME(destroy)
                    155: #define pmap_copy              PMAPNAME(copy)
                    156: #define pmap_update            PMAPNAME(update)
                    157: #define pmap_enter             PMAPNAME(enter)
                    158: #define pmap_remove            PMAPNAME(remove)
                    159: #define pmap_kenter_pa         PMAPNAME(kenter_pa)
                    160: #define pmap_kremove           PMAPNAME(kremove)
                    161: #define pmap_extract           PMAPNAME(extract)
                    162: #define pmap_protect           PMAPNAME(protect)
                    163: #define pmap_unwire            PMAPNAME(unwire)
                    164: #define pmap_page_protect      PMAPNAME(page_protect)
1.111   ! martin    165: #define        pmap_pv_protect         PMAPNAME(pv_protect)
1.53      garbled   166: #define pmap_query_bit         PMAPNAME(query_bit)
                    167: #define pmap_clear_bit         PMAPNAME(clear_bit)
                    168:
                    169: #define pmap_activate          PMAPNAME(activate)
                    170: #define pmap_deactivate                PMAPNAME(deactivate)
                    171:
                    172: #define pmap_pinit             PMAPNAME(pinit)
                    173: #define pmap_procwr            PMAPNAME(procwr)
                    174:
1.86      matt      175: #define pmap_pool              PMAPNAME(pool)
1.106     martin    176: #define pmap_pvo_pool          PMAPNAME(pvo_pool)
1.86      matt      177: #define pmap_pvo_table         PMAPNAME(pvo_table)
1.53      garbled   178: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
                    179: #define pmap_pte_print         PMAPNAME(pte_print)
                    180: #define pmap_pteg_check                PMAPNAME(pteg_check)
                    181: #define pmap_print_mmruregs    PMAPNAME(print_mmuregs)
                    182: #define pmap_print_pte         PMAPNAME(print_pte)
                    183: #define pmap_pteg_dist         PMAPNAME(pteg_dist)
                    184: #endif
                    185: #if defined(DEBUG) || defined(PMAPCHECK)
                    186: #define        pmap_pvo_verify         PMAPNAME(pvo_verify)
1.56      phx       187: #define pmapcheck              PMAPNAME(check)
                    188: #endif
                    189: #if defined(DEBUG) || defined(PMAPDEBUG)
                    190: #define pmapdebug              PMAPNAME(debug)
1.53      garbled   191: #endif
                    192: #define pmap_steal_memory      PMAPNAME(steal_memory)
                    193: #define pmap_bootstrap         PMAPNAME(bootstrap)
1.100     rin       194: #define pmap_bootstrap1                PMAPNAME(bootstrap1)
                    195: #define pmap_bootstrap2                PMAPNAME(bootstrap2)
1.53      garbled   196: #else
                    197: #define        STATIC                  /* nothing */
                    198: #endif /* PMAPNAME */
                    199:
                    200: STATIC int pmap_pte_spill(struct pmap *, vaddr_t, bool);
                    201: STATIC void pmap_real_memory(paddr_t *, psize_t *);
                    202: STATIC void pmap_init(void);
                    203: STATIC void pmap_virtual_space(vaddr_t *, vaddr_t *);
                    204: STATIC pmap_t pmap_create(void);
                    205: STATIC void pmap_reference(pmap_t);
                    206: STATIC void pmap_destroy(pmap_t);
                    207: STATIC void pmap_copy(pmap_t, pmap_t, vaddr_t, vsize_t, vaddr_t);
                    208: STATIC void pmap_update(pmap_t);
1.65      cegger    209: STATIC int pmap_enter(pmap_t, vaddr_t, paddr_t, vm_prot_t, u_int);
1.53      garbled   210: STATIC void pmap_remove(pmap_t, vaddr_t, vaddr_t);
1.68      cegger    211: STATIC void pmap_kenter_pa(vaddr_t, paddr_t, vm_prot_t, u_int);
1.53      garbled   212: STATIC void pmap_kremove(vaddr_t, vsize_t);
                    213: STATIC bool pmap_extract(pmap_t, vaddr_t, paddr_t *);
                    214:
                    215: STATIC void pmap_protect(pmap_t, vaddr_t, vaddr_t, vm_prot_t);
                    216: STATIC void pmap_unwire(pmap_t, vaddr_t);
                    217: STATIC void pmap_page_protect(struct vm_page *, vm_prot_t);
1.111   ! martin    218: STATIC void pmap_pv_protect(paddr_t, vm_prot_t);
1.53      garbled   219: STATIC bool pmap_query_bit(struct vm_page *, int);
                    220: STATIC bool pmap_clear_bit(struct vm_page *, int);
                    221:
                    222: STATIC void pmap_activate(struct lwp *);
                    223: STATIC void pmap_deactivate(struct lwp *);
                    224:
                    225: STATIC void pmap_pinit(pmap_t pm);
                    226: STATIC void pmap_procwr(struct proc *, vaddr_t, size_t);
                    227:
                    228: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
                    229: STATIC void pmap_pte_print(volatile struct pte *);
                    230: STATIC void pmap_pteg_check(void);
                    231: STATIC void pmap_print_mmuregs(void);
                    232: STATIC void pmap_print_pte(pmap_t, vaddr_t);
                    233: STATIC void pmap_pteg_dist(void);
                    234: #endif
                    235: #if defined(DEBUG) || defined(PMAPCHECK)
                    236: STATIC void pmap_pvo_verify(void);
                    237: #endif
                    238: STATIC vaddr_t pmap_steal_memory(vsize_t, vaddr_t *, vaddr_t *);
                    239: STATIC void pmap_bootstrap(paddr_t, paddr_t);
1.100     rin       240: STATIC void pmap_bootstrap1(paddr_t, paddr_t);
                    241: STATIC void pmap_bootstrap2(void);
1.53      garbled   242:
                    243: #ifdef PMAPNAME
                    244: const struct pmap_ops PMAPNAME(ops) = {
                    245:        .pmapop_pte_spill = pmap_pte_spill,
                    246:        .pmapop_real_memory = pmap_real_memory,
                    247:        .pmapop_init = pmap_init,
                    248:        .pmapop_virtual_space = pmap_virtual_space,
                    249:        .pmapop_create = pmap_create,
                    250:        .pmapop_reference = pmap_reference,
                    251:        .pmapop_destroy = pmap_destroy,
                    252:        .pmapop_copy = pmap_copy,
                    253:        .pmapop_update = pmap_update,
                    254:        .pmapop_enter = pmap_enter,
                    255:        .pmapop_remove = pmap_remove,
                    256:        .pmapop_kenter_pa = pmap_kenter_pa,
                    257:        .pmapop_kremove = pmap_kremove,
                    258:        .pmapop_extract = pmap_extract,
                    259:        .pmapop_protect = pmap_protect,
                    260:        .pmapop_unwire = pmap_unwire,
                    261:        .pmapop_page_protect = pmap_page_protect,
1.111   ! martin    262:        .pmapop_pv_protect = pmap_pv_protect,
1.53      garbled   263:        .pmapop_query_bit = pmap_query_bit,
                    264:        .pmapop_clear_bit = pmap_clear_bit,
                    265:        .pmapop_activate = pmap_activate,
                    266:        .pmapop_deactivate = pmap_deactivate,
                    267:        .pmapop_pinit = pmap_pinit,
                    268:        .pmapop_procwr = pmap_procwr,
                    269: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
                    270:        .pmapop_pte_print = pmap_pte_print,
                    271:        .pmapop_pteg_check = pmap_pteg_check,
                    272:        .pmapop_print_mmuregs = pmap_print_mmuregs,
                    273:        .pmapop_print_pte = pmap_print_pte,
                    274:        .pmapop_pteg_dist = pmap_pteg_dist,
                    275: #else
                    276:        .pmapop_pte_print = NULL,
                    277:        .pmapop_pteg_check = NULL,
                    278:        .pmapop_print_mmuregs = NULL,
                    279:        .pmapop_print_pte = NULL,
                    280:        .pmapop_pteg_dist = NULL,
                    281: #endif
                    282: #if defined(DEBUG) || defined(PMAPCHECK)
                    283:        .pmapop_pvo_verify = pmap_pvo_verify,
                    284: #else
                    285:        .pmapop_pvo_verify = NULL,
1.1       matt      286: #endif
1.53      garbled   287:        .pmapop_steal_memory = pmap_steal_memory,
                    288:        .pmapop_bootstrap = pmap_bootstrap,
1.101     thorpej   289:        .pmapop_bootstrap1 = pmap_bootstrap1,
                    290:        .pmapop_bootstrap2 = pmap_bootstrap2,
1.53      garbled   291: };
                    292: #endif /* !PMAPNAME */
1.1       matt      293:
                    294: /*
1.38      sanjayl   295:  * The following structure is aligned to 32 bytes
1.1       matt      296:  */
                    297: struct pvo_entry {
                    298:        LIST_ENTRY(pvo_entry) pvo_vlink;        /* Link to common virt page */
                    299:        TAILQ_ENTRY(pvo_entry) pvo_olink;       /* Link to overflow entry */
                    300:        struct pte pvo_pte;                     /* Prebuilt PTE */
                    301:        pmap_t pvo_pmap;                        /* ptr to owning pmap */
                    302:        vaddr_t pvo_vaddr;                      /* VA of entry */
                    303: #define        PVO_PTEGIDX_MASK        0x0007          /* which PTEG slot */
                    304: #define        PVO_PTEGIDX_VALID       0x0008          /* slot is valid */
                    305: #define        PVO_WIRED               0x0010          /* PVO entry is wired */
                    306: #define        PVO_MANAGED             0x0020          /* PVO e. for managed page */
                    307: #define        PVO_EXECUTABLE          0x0040          /* PVO e. for executable page */
1.39      matt      308: #define        PVO_WIRED_P(pvo)        ((pvo)->pvo_vaddr & PVO_WIRED)
                    309: #define        PVO_MANAGED_P(pvo)      ((pvo)->pvo_vaddr & PVO_MANAGED)
                    310: #define        PVO_EXECUTABLE_P(pvo)   ((pvo)->pvo_vaddr & PVO_EXECUTABLE)
1.12      matt      311: #define        PVO_ENTER_INSERT        0               /* PVO has been removed */
                    312: #define        PVO_SPILL_UNSET         1               /* PVO has been evicted */
                    313: #define        PVO_SPILL_SET           2               /* PVO has been spilled */
                    314: #define        PVO_SPILL_INSERT        3               /* PVO has been inserted */
                    315: #define        PVO_PMAP_PAGE_PROTECT   4               /* PVO has changed */
                    316: #define        PVO_PMAP_PROTECT        5               /* PVO has changed */
                    317: #define        PVO_REMOVE              6               /* PVO has been removed */
                    318: #define        PVO_WHERE_MASK          15
                    319: #define        PVO_WHERE_SHFT          8
1.38      sanjayl   320: } __attribute__ ((aligned (32)));
1.1       matt      321: #define        PVO_VADDR(pvo)          ((pvo)->pvo_vaddr & ~ADDR_POFF)
                    322: #define        PVO_PTEGIDX_GET(pvo)    ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
                    323: #define        PVO_PTEGIDX_ISSET(pvo)  ((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID)
                    324: #define        PVO_PTEGIDX_CLR(pvo)    \
                    325:        ((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK)))
                    326: #define        PVO_PTEGIDX_SET(pvo,i)  \
                    327:        ((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID))
1.12      matt      328: #define        PVO_WHERE(pvo,w)        \
                    329:        ((pvo)->pvo_vaddr &= ~(PVO_WHERE_MASK << PVO_WHERE_SHFT), \
                    330:         (pvo)->pvo_vaddr |= ((PVO_ ## w) << PVO_WHERE_SHFT))
1.1       matt      331:
                    332: TAILQ_HEAD(pvo_tqhead, pvo_entry);
                    333: struct pvo_tqhead *pmap_pvo_table;     /* pvo entries by ptegroup index */
                    334:
                    335: struct pool pmap_pool;         /* pool for pmap structures */
1.106     martin    336: struct pool pmap_pvo_pool;     /* pool for pvo entries */
1.1       matt      337:
                    338: /*
                    339:  * We keep a cache of unmanaged pages to be used for pvo entries for
                    340:  * unmanaged pages.
                    341:  */
                    342: struct pvo_page {
                    343:        SIMPLEQ_ENTRY(pvo_page) pvop_link;
                    344: };
                    345: SIMPLEQ_HEAD(pvop_head, pvo_page);
1.106     martin    346: static struct pvop_head pmap_pvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_pvop_head);
                    347: static u_long pmap_pvop_free;
                    348: static u_long pmap_pvop_maxfree;
                    349:
                    350: static void *pmap_pool_alloc(struct pool *, int);
                    351: static void pmap_pool_free(struct pool *, void *);
                    352:
                    353: static struct pool_allocator pmap_pool_allocator = {
                    354:        .pa_alloc = pmap_pool_alloc,
                    355:        .pa_free = pmap_pool_free,
1.43      garbled   356:        .pa_pagesz = 0,
1.1       matt      357: };
                    358:
                    359: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
1.2       matt      360: void pmap_pte_print(volatile struct pte *);
1.1       matt      361: void pmap_pteg_check(void);
                    362: void pmap_pteg_dist(void);
                    363: void pmap_print_pte(pmap_t, vaddr_t);
                    364: void pmap_print_mmuregs(void);
                    365: #endif
                    366:
                    367: #if defined(DEBUG) || defined(PMAPCHECK)
                    368: #ifdef PMAPCHECK
                    369: int pmapcheck = 1;
                    370: #else
                    371: int pmapcheck = 0;
                    372: #endif
                    373: void pmap_pvo_verify(void);
1.53      garbled   374: static void pmap_pvo_check(const struct pvo_entry *);
1.1       matt      375: #define        PMAP_PVO_CHECK(pvo)                     \
                    376:        do {                                    \
                    377:                if (pmapcheck)                  \
                    378:                        pmap_pvo_check(pvo);    \
                    379:        } while (0)
                    380: #else
                    381: #define        PMAP_PVO_CHECK(pvo)     do { } while (/*CONSTCOND*/0)
                    382: #endif
1.53      garbled   383: static int pmap_pte_insert(int, struct pte *);
                    384: static int pmap_pvo_enter(pmap_t, struct pool *, struct pvo_head *,
1.2       matt      385:        vaddr_t, paddr_t, register_t, int);
1.53      garbled   386: static void pmap_pvo_remove(struct pvo_entry *, int, struct pvo_head *);
                    387: static void pmap_pvo_free(struct pvo_entry *);
                    388: static void pmap_pvo_free_list(struct pvo_head *);
                    389: static struct pvo_entry *pmap_pvo_find_va(pmap_t, vaddr_t, int *);
                    390: static volatile struct pte *pmap_pvo_to_pte(const struct pvo_entry *, int);
                    391: static struct pvo_entry *pmap_pvo_reclaim(struct pmap *);
                    392: static void pvo_set_exec(struct pvo_entry *);
                    393: static void pvo_clear_exec(struct pvo_entry *);
1.1       matt      394:
1.53      garbled   395: static void tlbia(void);
1.1       matt      396:
1.53      garbled   397: static void pmap_release(pmap_t);
                    398: static paddr_t pmap_boot_find_memory(psize_t, psize_t, int);
1.1       matt      399:
1.25      chs       400: static uint32_t pmap_pvo_reclaim_nextidx;
                    401: #ifdef DEBUG
                    402: static int pmap_pvo_reclaim_debugctr;
                    403: #endif
                    404:
1.1       matt      405: #define        VSID_NBPW       (sizeof(uint32_t) * 8)
                    406: static uint32_t pmap_vsid_bitmap[NPMAPS / VSID_NBPW];
                    407:
                    408: static int pmap_initialized;
                    409:
                    410: #if defined(DEBUG) || defined(PMAPDEBUG)
                    411: #define        PMAPDEBUG_BOOT          0x0001
                    412: #define        PMAPDEBUG_PTE           0x0002
                    413: #define        PMAPDEBUG_EXEC          0x0008
                    414: #define        PMAPDEBUG_PVOENTER      0x0010
                    415: #define        PMAPDEBUG_PVOREMOVE     0x0020
                    416: #define        PMAPDEBUG_ACTIVATE      0x0100
                    417: #define        PMAPDEBUG_CREATE        0x0200
                    418: #define        PMAPDEBUG_ENTER         0x1000
                    419: #define        PMAPDEBUG_KENTER        0x2000
                    420: #define        PMAPDEBUG_KREMOVE       0x4000
                    421: #define        PMAPDEBUG_REMOVE        0x8000
1.38      sanjayl   422:
1.1       matt      423: unsigned int pmapdebug = 0;
1.38      sanjayl   424:
1.85      matt      425: # define DPRINTF(x, ...)       printf(x, __VA_ARGS__)
                    426: # define DPRINTFN(n, x, ...)   do if (pmapdebug & PMAPDEBUG_ ## n) printf(x, __VA_ARGS__); while (0)
1.1       matt      427: #else
1.85      matt      428: # define DPRINTF(x, ...)       do { } while (0)
                    429: # define DPRINTFN(n, x, ...)   do { } while (0)
1.1       matt      430: #endif
                    431:
                    432:
                    433: #ifdef PMAPCOUNTERS
                    434: /*
                    435:  * From pmap_subr.c
                    436:  */
1.53      garbled   437: extern struct evcnt pmap_evcnt_mappings;
                    438: extern struct evcnt pmap_evcnt_unmappings;
                    439:
                    440: extern struct evcnt pmap_evcnt_kernel_mappings;
                    441: extern struct evcnt pmap_evcnt_kernel_unmappings;
                    442:
                    443: extern struct evcnt pmap_evcnt_mappings_replaced;
                    444:
                    445: extern struct evcnt pmap_evcnt_exec_mappings;
                    446: extern struct evcnt pmap_evcnt_exec_cached;
                    447:
                    448: extern struct evcnt pmap_evcnt_exec_synced;
                    449: extern struct evcnt pmap_evcnt_exec_synced_clear_modify;
                    450: extern struct evcnt pmap_evcnt_exec_synced_pvo_remove;
                    451:
                    452: extern struct evcnt pmap_evcnt_exec_uncached_page_protect;
                    453: extern struct evcnt pmap_evcnt_exec_uncached_clear_modify;
                    454: extern struct evcnt pmap_evcnt_exec_uncached_zero_page;
                    455: extern struct evcnt pmap_evcnt_exec_uncached_copy_page;
                    456: extern struct evcnt pmap_evcnt_exec_uncached_pvo_remove;
                    457:
                    458: extern struct evcnt pmap_evcnt_updates;
                    459: extern struct evcnt pmap_evcnt_collects;
                    460: extern struct evcnt pmap_evcnt_copies;
                    461:
                    462: extern struct evcnt pmap_evcnt_ptes_spilled;
                    463: extern struct evcnt pmap_evcnt_ptes_unspilled;
                    464: extern struct evcnt pmap_evcnt_ptes_evicted;
                    465:
                    466: extern struct evcnt pmap_evcnt_ptes_primary[8];
                    467: extern struct evcnt pmap_evcnt_ptes_secondary[8];
                    468: extern struct evcnt pmap_evcnt_ptes_removed;
                    469: extern struct evcnt pmap_evcnt_ptes_changed;
                    470: extern struct evcnt pmap_evcnt_pvos_reclaimed;
                    471: extern struct evcnt pmap_evcnt_pvos_failed;
                    472:
1.1       matt      473: extern struct evcnt pmap_evcnt_zeroed_pages;
                    474: extern struct evcnt pmap_evcnt_copied_pages;
                    475: extern struct evcnt pmap_evcnt_idlezeroed_pages;
1.26      matt      476:
1.53      garbled   477: #define        PMAPCOUNT(ev)   ((pmap_evcnt_ ## ev).ev_count++)
                    478: #define        PMAPCOUNT2(ev)  ((ev).ev_count++)
1.1       matt      479: #else
                    480: #define        PMAPCOUNT(ev)   ((void) 0)
                    481: #define        PMAPCOUNT2(ev)  ((void) 0)
                    482: #endif
                    483:
1.109     riastrad  484: #define        TLBIE(va)       __asm volatile("tlbie %0" :: "r"(va) : "memory")
1.38      sanjayl   485:
                    486: /* XXXSL: this needs to be moved to assembler */
1.109     riastrad  487: #define        TLBIEL(va)      __asm volatile("tlbie %0" :: "r"(va) : "memory")
1.38      sanjayl   488:
1.87      kiyohara  489: #ifdef MD_TLBSYNC
                    490: #define TLBSYNC()      MD_TLBSYNC()
                    491: #else
1.109     riastrad  492: #define        TLBSYNC()       __asm volatile("tlbsync" ::: "memory")
1.87      kiyohara  493: #endif
1.109     riastrad  494: #define        SYNC()          __asm volatile("sync" ::: "memory")
                    495: #define        EIEIO()         __asm volatile("eieio" ::: "memory")
                    496: #define        DCBST(va)       __asm volatile("dcbst 0,%0" :: "r"(va) : "memory")
1.1       matt      497: #define        MFMSR()         mfmsr()
                    498: #define        MTMSR(psl)      mtmsr(psl)
                    499: #define        MFPVR()         mfpvr()
                    500: #define        MFSRIN(va)      mfsrin(va)
                    501: #define        MFTB()          mfrtcltbl()
                    502:
1.92      joerg     503: #if defined(DDB) && !defined(PMAP_OEA64)
1.35      perry     504: static inline register_t
1.1       matt      505: mfsrin(vaddr_t va)
                    506: {
1.2       matt      507:        register_t sr;
1.35      perry     508:        __asm volatile ("mfsrin %0,%1" : "=r"(sr) : "r"(va));
1.1       matt      509:        return sr;
                    510: }
1.92      joerg     511: #endif /* DDB && !PMAP_OEA64 */
1.38      sanjayl   512:
1.53      garbled   513: #if defined (PMAP_OEA64_BRIDGE)
1.38      sanjayl   514: extern void mfmsr64 (register64_t *result);
1.53      garbled   515: #endif /* PMAP_OEA64_BRIDGE */
1.38      sanjayl   516:
1.50      ad        517: #define        PMAP_LOCK()             KERNEL_LOCK(1, NULL)
                    518: #define        PMAP_UNLOCK()           KERNEL_UNLOCK_ONE(NULL)
1.1       matt      519:
1.35      perry     520: static inline register_t
1.1       matt      521: pmap_interrupts_off(void)
                    522: {
1.2       matt      523:        register_t msr = MFMSR();
1.1       matt      524:        if (msr & PSL_EE)
                    525:                MTMSR(msr & ~PSL_EE);
                    526:        return msr;
                    527: }
                    528:
                    529: static void
1.2       matt      530: pmap_interrupts_restore(register_t msr)
1.1       matt      531: {
                    532:        if (msr & PSL_EE)
                    533:                MTMSR(msr);
                    534: }
                    535:
1.35      perry     536: static inline u_int32_t
1.1       matt      537: mfrtcltbl(void)
                    538: {
1.55      garbled   539: #ifdef PPC_OEA601
1.1       matt      540:        if ((MFPVR() >> 16) == MPC601)
                    541:                return (mfrtcl() >> 7);
                    542:        else
1.55      garbled   543: #endif
1.1       matt      544:                return (mftbl());
                    545: }
                    546:
                    547: /*
                    548:  * These small routines may have to be replaced,
                    549:  * if/when we support processors other that the 604.
                    550:  */
                    551:
                    552: void
                    553: tlbia(void)
                    554: {
1.47      macallan  555:        char *i;
1.1       matt      556:
                    557:        SYNC();
1.53      garbled   558: #if defined(PMAP_OEA)
1.1       matt      559:        /*
                    560:         * Why not use "tlbia"?  Because not all processors implement it.
                    561:         *
1.20      wiz       562:         * This needs to be a per-CPU callback to do the appropriate thing
1.1       matt      563:         * for the CPU. XXX
                    564:         */
1.47      macallan  565:        for (i = 0; i < (char *)0x00040000; i += 0x00001000) {
1.1       matt      566:                TLBIE(i);
                    567:                EIEIO();
                    568:                SYNC();
                    569:        }
1.53      garbled   570: #elif defined (PMAP_OEA64) || defined (PMAP_OEA64_BRIDGE)
1.38      sanjayl   571:        /* This is specifically for the 970, 970UM v1.6 pp. 140. */
1.51      garbled   572:        for (i = 0; i <= (char *)0xFF000; i += 0x00001000) {
1.38      sanjayl   573:                TLBIEL(i);
                    574:                EIEIO();
                    575:                SYNC();
                    576:        }
                    577: #endif
1.1       matt      578:        TLBSYNC();
                    579:        SYNC();
                    580: }
                    581:
1.35      perry     582: static inline register_t
1.2       matt      583: va_to_vsid(const struct pmap *pm, vaddr_t addr)
1.1       matt      584: {
1.18      matt      585:        /*
1.102     thorpej   586:         * Rather than searching the STE groups for the VSID or extracting
                    587:         * it from the SR, we know how we generate that from the ESID and
                    588:         * so do that.
                    589:         *
                    590:         * This makes the code the same for OEA and OEA64, and also allows
                    591:         * us to generate a correct-for-that-address-space VSID even if the
                    592:         * pmap contains a different SR value at any given moment (e.g.
                    593:         * kernel pmap on a 601 that is using I/O segments).
1.18      matt      594:         */
                    595:        return VSID_MAKE(addr >> ADDR_SR_SHFT, pm->pm_vsid) >> SR_VSID_SHFT;
1.1       matt      596: }
                    597:
1.35      perry     598: static inline register_t
1.2       matt      599: va_to_pteg(const struct pmap *pm, vaddr_t addr)
1.1       matt      600: {
1.2       matt      601:        register_t hash;
                    602:
                    603:        hash = va_to_vsid(pm, addr) ^ ((addr & ADDR_PIDX) >> ADDR_PIDX_SHFT);
1.1       matt      604:        return hash & pmap_pteg_mask;
                    605: }
                    606:
                    607: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
                    608: /*
                    609:  * Given a PTE in the page table, calculate the VADDR that hashes to it.
                    610:  * The only bit of magic is that the top 4 bits of the address doesn't
                    611:  * technically exist in the PTE.  But we know we reserved 4 bits of the
                    612:  * VSID for it so that's how we get it.
                    613:  */
                    614: static vaddr_t
1.2       matt      615: pmap_pte_to_va(volatile const struct pte *pt)
1.1       matt      616: {
                    617:        vaddr_t va;
                    618:        uintptr_t ptaddr = (uintptr_t) pt;
                    619:
                    620:        if (pt->pte_hi & PTE_HID)
1.2       matt      621:                ptaddr ^= (pmap_pteg_mask * sizeof(struct pteg));
1.1       matt      622:
1.18      matt      623:        /* PPC Bits 10-19  PPC64 Bits 42-51 */
1.53      garbled   624: #if defined(PMAP_OEA)
1.4       matt      625:        va = ((pt->pte_hi >> PTE_VSID_SHFT) ^ (ptaddr / sizeof(struct pteg))) & 0x3ff;
1.53      garbled   626: #elif defined (PMAP_OEA64) || defined (PMAP_OEA64_BRIDGE)
1.38      sanjayl   627:        va = ((pt->pte_hi >> PTE_VSID_SHFT) ^ (ptaddr / sizeof(struct pteg))) & 0x7ff;
                    628: #endif
1.1       matt      629:        va <<= ADDR_PIDX_SHFT;
                    630:
1.18      matt      631:        /* PPC Bits 4-9  PPC64 Bits 36-41 */
1.1       matt      632:        va |= (pt->pte_hi & PTE_API) << ADDR_API_SHFT;
                    633:
1.53      garbled   634: #if defined(PMAP_OEA64)
1.18      matt      635:        /* PPC63 Bits 0-35 */
                    636:        /* va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT; */
1.53      garbled   637: #elif defined(PMAP_OEA) || defined(PMAP_OEA64_BRIDGE)
1.1       matt      638:        /* PPC Bits 0-3 */
                    639:        va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT;
1.18      matt      640: #endif
1.1       matt      641:
                    642:        return va;
                    643: }
                    644: #endif
                    645:
1.35      perry     646: static inline struct pvo_head *
1.1       matt      647: pa_to_pvoh(paddr_t pa, struct vm_page **pg_p)
                    648: {
                    649:        struct vm_page *pg;
1.72      uebayasi  650:        struct vm_page_md *md;
1.108     riastrad  651:        struct pmap_page *pp;
1.1       matt      652:
                    653:        pg = PHYS_TO_VM_PAGE(pa);
                    654:        if (pg_p != NULL)
                    655:                *pg_p = pg;
1.108     riastrad  656:        if (pg == NULL) {
                    657:                if ((pp = pmap_pv_tracked(pa)) != NULL)
                    658:                        return &pp->pp_pvoh;
1.107     chs       659:                return NULL;
1.108     riastrad  660:        }
1.72      uebayasi  661:        md = VM_PAGE_TO_MD(pg);
                    662:        return &md->mdpg_pvoh;
1.1       matt      663: }
                    664:
1.35      perry     665: static inline struct pvo_head *
1.1       matt      666: vm_page_to_pvoh(struct vm_page *pg)
                    667: {
1.72      uebayasi  668:        struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
                    669:
                    670:        return &md->mdpg_pvoh;
1.1       matt      671: }
                    672:
1.108     riastrad  673: static inline void
                    674: pmap_pp_attr_clear(struct pmap_page *pp, int ptebit)
                    675: {
                    676:
                    677:        pp->pp_attrs &= ptebit;
                    678: }
1.1       matt      679:
1.35      perry     680: static inline void
1.1       matt      681: pmap_attr_clear(struct vm_page *pg, int ptebit)
                    682: {
1.72      uebayasi  683:        struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
                    684:
1.108     riastrad  685:        pmap_pp_attr_clear(&md->mdpg_pp, ptebit);
                    686: }
                    687:
                    688: static inline int
                    689: pmap_pp_attr_fetch(struct pmap_page *pp)
                    690: {
                    691:
                    692:        return pp->pp_attrs;
1.1       matt      693: }
                    694:
1.35      perry     695: static inline int
1.1       matt      696: pmap_attr_fetch(struct vm_page *pg)
                    697: {
1.72      uebayasi  698:        struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
                    699:
1.108     riastrad  700:        return pmap_pp_attr_fetch(&md->mdpg_pp);
1.1       matt      701: }
                    702:
1.35      perry     703: static inline void
1.1       matt      704: pmap_attr_save(struct vm_page *pg, int ptebit)
                    705: {
1.72      uebayasi  706:        struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
                    707:
                    708:        md->mdpg_attrs |= ptebit;
1.1       matt      709: }
                    710:
1.35      perry     711: static inline int
1.2       matt      712: pmap_pte_compare(const volatile struct pte *pt, const struct pte *pvo_pt)
1.1       matt      713: {
                    714:        if (pt->pte_hi == pvo_pt->pte_hi
                    715: #if 0
                    716:            && ((pt->pte_lo ^ pvo_pt->pte_lo) &
                    717:                ~(PTE_REF|PTE_CHG)) == 0
                    718: #endif
                    719:            )
                    720:                return 1;
                    721:        return 0;
                    722: }
                    723:
1.35      perry     724: static inline void
1.2       matt      725: pmap_pte_create(struct pte *pt, const struct pmap *pm, vaddr_t va, register_t pte_lo)
1.1       matt      726: {
                    727:        /*
                    728:         * Construct the PTE.  Default to IMB initially.  Valid bit
                    729:         * only gets set when the real pte is set in memory.
                    730:         *
                    731:         * Note: Don't set the valid bit for correct operation of tlb update.
                    732:         */
1.53      garbled   733: #if defined(PMAP_OEA)
1.2       matt      734:        pt->pte_hi = (va_to_vsid(pm, va) << PTE_VSID_SHFT)
                    735:            | (((va & ADDR_PIDX) >> (ADDR_API_SHFT - PTE_API_SHFT)) & PTE_API);
1.1       matt      736:        pt->pte_lo = pte_lo;
1.79      matt      737: #elif defined (PMAP_OEA64_BRIDGE) || defined (PMAP_OEA64)
1.38      sanjayl   738:        pt->pte_hi = ((u_int64_t)va_to_vsid(pm, va) << PTE_VSID_SHFT)
                    739:            | (((va & ADDR_PIDX) >> (ADDR_API_SHFT - PTE_API_SHFT)) & PTE_API);
                    740:        pt->pte_lo = (u_int64_t) pte_lo;
1.53      garbled   741: #endif /* PMAP_OEA */
1.1       matt      742: }
                    743:
1.35      perry     744: static inline void
1.2       matt      745: pmap_pte_synch(volatile struct pte *pt, struct pte *pvo_pt)
1.1       matt      746: {
                    747:        pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF|PTE_CHG);
                    748: }
                    749:
1.35      perry     750: static inline void
1.2       matt      751: pmap_pte_clear(volatile struct pte *pt, vaddr_t va, int ptebit)
1.1       matt      752: {
                    753:        /*
                    754:         * As shown in Section 7.6.3.2.3
                    755:         */
                    756:        pt->pte_lo &= ~ptebit;
                    757:        TLBIE(va);
                    758:        SYNC();
                    759:        EIEIO();
                    760:        TLBSYNC();
                    761:        SYNC();
1.57      matt      762: #ifdef MULTIPROCESSOR
                    763:        DCBST(pt);
                    764: #endif
1.1       matt      765: }
                    766:
1.35      perry     767: static inline void
1.2       matt      768: pmap_pte_set(volatile struct pte *pt, struct pte *pvo_pt)
1.1       matt      769: {
                    770: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
                    771:        if (pvo_pt->pte_hi & PTE_VALID)
                    772:                panic("pte_set: setting an already valid pte %p", pvo_pt);
                    773: #endif
                    774:        pvo_pt->pte_hi |= PTE_VALID;
1.38      sanjayl   775:
1.1       matt      776:        /*
                    777:         * Update the PTE as defined in section 7.6.3.1
                    778:         * Note that the REF/CHG bits are from pvo_pt and thus should
                    779:         * have been saved so this routine can restore them (if desired).
                    780:         */
                    781:        pt->pte_lo = pvo_pt->pte_lo;
                    782:        EIEIO();
                    783:        pt->pte_hi = pvo_pt->pte_hi;
1.38      sanjayl   784:        TLBSYNC();
1.1       matt      785:        SYNC();
1.57      matt      786: #ifdef MULTIPROCESSOR
                    787:        DCBST(pt);
                    788: #endif
1.1       matt      789:        pmap_pte_valid++;
                    790: }
                    791:
1.35      perry     792: static inline void
1.2       matt      793: pmap_pte_unset(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va)
1.1       matt      794: {
                    795: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
                    796:        if ((pvo_pt->pte_hi & PTE_VALID) == 0)
                    797:                panic("pte_unset: attempt to unset an inactive pte#1 %p/%p", pvo_pt, pt);
                    798:        if ((pt->pte_hi & PTE_VALID) == 0)
                    799:                panic("pte_unset: attempt to unset an inactive pte#2 %p/%p", pvo_pt, pt);
                    800: #endif
                    801:
                    802:        pvo_pt->pte_hi &= ~PTE_VALID;
                    803:        /*
                    804:         * Force the ref & chg bits back into the PTEs.
                    805:         */
                    806:        SYNC();
                    807:        /*
                    808:         * Invalidate the pte ... (Section 7.6.3.3)
                    809:         */
                    810:        pt->pte_hi &= ~PTE_VALID;
                    811:        SYNC();
                    812:        TLBIE(va);
                    813:        SYNC();
                    814:        EIEIO();
                    815:        TLBSYNC();
                    816:        SYNC();
                    817:        /*
                    818:         * Save the ref & chg bits ...
                    819:         */
                    820:        pmap_pte_synch(pt, pvo_pt);
                    821:        pmap_pte_valid--;
                    822: }
                    823:
1.35      perry     824: static inline void
1.2       matt      825: pmap_pte_change(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va)
1.1       matt      826: {
                    827:        /*
                    828:         * Invalidate the PTE
                    829:         */
                    830:        pmap_pte_unset(pt, pvo_pt, va);
                    831:        pmap_pte_set(pt, pvo_pt);
                    832: }
                    833:
                    834: /*
                    835:  * Try to insert the PTE @ *pvo_pt into the pmap_pteg_table at ptegidx
                    836:  * (either primary or secondary location).
                    837:  *
                    838:  * Note: both the destination and source PTEs must not have PTE_VALID set.
                    839:  */
                    840:
1.53      garbled   841: static int
1.2       matt      842: pmap_pte_insert(int ptegidx, struct pte *pvo_pt)
1.1       matt      843: {
1.2       matt      844:        volatile struct pte *pt;
1.1       matt      845:        int i;
                    846:
                    847: #if defined(DEBUG)
1.85      matt      848:        DPRINTFN(PTE, "pmap_pte_insert: idx %#x, pte %#" _PRIxpte " %#" _PRIxpte "\n",
                    849:                ptegidx, pvo_pt->pte_hi, pvo_pt->pte_lo);
1.1       matt      850: #endif
                    851:        /*
                    852:         * First try primary hash.
                    853:         */
                    854:        for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
                    855:                if ((pt->pte_hi & PTE_VALID) == 0) {
                    856:                        pvo_pt->pte_hi &= ~PTE_HID;
                    857:                        pmap_pte_set(pt, pvo_pt);
                    858:                        return i;
                    859:                }
                    860:        }
                    861:
                    862:        /*
                    863:         * Now try secondary hash.
                    864:         */
                    865:        ptegidx ^= pmap_pteg_mask;
                    866:        for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
                    867:                if ((pt->pte_hi & PTE_VALID) == 0) {
                    868:                        pvo_pt->pte_hi |= PTE_HID;
                    869:                        pmap_pte_set(pt, pvo_pt);
                    870:                        return i;
                    871:                }
                    872:        }
                    873:        return -1;
                    874: }
                    875:
                    876: /*
                    877:  * Spill handler.
                    878:  *
                    879:  * Tries to spill a page table entry from the overflow area.
                    880:  * This runs in either real mode (if dealing with a exception spill)
                    881:  * or virtual mode when dealing with manually spilling one of the
                    882:  * kernel's pte entries.  In either case, interrupts are already
                    883:  * disabled.
                    884:  */
1.14      chs       885:
1.1       matt      886: int
1.44      thorpej   887: pmap_pte_spill(struct pmap *pm, vaddr_t addr, bool exec)
1.1       matt      888: {
                    889:        struct pvo_entry *source_pvo, *victim_pvo, *next_pvo;
                    890:        struct pvo_entry *pvo;
1.15      dyoung    891:        /* XXX: gcc -- vpvoh is always set at either *1* or *2* */
                    892:        struct pvo_tqhead *pvoh, *vpvoh = NULL;
1.1       matt      893:        int ptegidx, i, j;
1.2       matt      894:        volatile struct pteg *pteg;
                    895:        volatile struct pte *pt;
1.1       matt      896:
1.50      ad        897:        PMAP_LOCK();
                    898:
1.2       matt      899:        ptegidx = va_to_pteg(pm, addr);
1.1       matt      900:
                    901:        /*
                    902:         * Have to substitute some entry. Use the primary hash for this.
1.12      matt      903:         * Use low bits of timebase as random generator.  Make sure we are
                    904:         * not picking a kernel pte for replacement.
1.1       matt      905:         */
                    906:        pteg = &pmap_pteg_table[ptegidx];
                    907:        i = MFTB() & 7;
1.12      matt      908:        for (j = 0; j < 8; j++) {
                    909:                pt = &pteg->pt[i];
1.53      garbled   910:                if ((pt->pte_hi & PTE_VALID) == 0)
                    911:                        break;
                    912:                if (VSID_TO_HASH((pt->pte_hi & PTE_VSID) >> PTE_VSID_SHFT)
                    913:                                < PHYSMAP_VSIDBITS)
1.12      matt      914:                        break;
                    915:                i = (i + 1) & 7;
                    916:        }
                    917:        KASSERT(j < 8);
1.1       matt      918:
                    919:        source_pvo = NULL;
                    920:        victim_pvo = NULL;
                    921:        pvoh = &pmap_pvo_table[ptegidx];
                    922:        TAILQ_FOREACH(pvo, pvoh, pvo_olink) {
                    923:
                    924:                /*
                    925:                 * We need to find pvo entry for this address...
                    926:                 */
                    927:                PMAP_PVO_CHECK(pvo);            /* sanity check */
                    928:
                    929:                /*
                    930:                 * If we haven't found the source and we come to a PVO with
                    931:                 * a valid PTE, then we know we can't find it because all
                    932:                 * evicted PVOs always are first in the list.
                    933:                 */
                    934:                if (source_pvo == NULL && (pvo->pvo_pte.pte_hi & PTE_VALID))
                    935:                        break;
1.2       matt      936:                if (source_pvo == NULL && pm == pvo->pvo_pmap &&
                    937:                    addr == PVO_VADDR(pvo)) {
1.1       matt      938:
                    939:                        /*
                    940:                         * Now we have found the entry to be spilled into the
                    941:                         * pteg.  Attempt to insert it into the page table.
                    942:                         */
                    943:                        j = pmap_pte_insert(ptegidx, &pvo->pvo_pte);
                    944:                        if (j >= 0) {
                    945:                                PVO_PTEGIDX_SET(pvo, j);
                    946:                                PMAP_PVO_CHECK(pvo);    /* sanity check */
1.12      matt      947:                                PVO_WHERE(pvo, SPILL_INSERT);
1.1       matt      948:                                pvo->pvo_pmap->pm_evictions--;
                    949:                                PMAPCOUNT(ptes_spilled);
                    950:                                PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID)
                    951:                                    ? pmap_evcnt_ptes_secondary
                    952:                                    : pmap_evcnt_ptes_primary)[j]);
                    953:
                    954:                                /*
                    955:                                 * Since we keep the evicted entries at the
                    956:                                 * from of the PVO list, we need move this
                    957:                                 * (now resident) PVO after the evicted
                    958:                                 * entries.
                    959:                                 */
                    960:                                next_pvo = TAILQ_NEXT(pvo, pvo_olink);
                    961:
                    962:                                /*
1.5       matt      963:                                 * If we don't have to move (either we were the
                    964:                                 * last entry or the next entry was valid),
1.1       matt      965:                                 * don't change our position.  Otherwise
                    966:                                 * move ourselves to the tail of the queue.
                    967:                                 */
                    968:                                if (next_pvo != NULL &&
                    969:                                    !(next_pvo->pvo_pte.pte_hi & PTE_VALID)) {
                    970:                                        TAILQ_REMOVE(pvoh, pvo, pvo_olink);
                    971:                                        TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink);
                    972:                                }
1.50      ad        973:                                PMAP_UNLOCK();
1.1       matt      974:                                return 1;
                    975:                        }
                    976:                        source_pvo = pvo;
1.39      matt      977:                        if (exec && !PVO_EXECUTABLE_P(source_pvo)) {
1.96      rin       978:                                PMAP_UNLOCK();
1.14      chs       979:                                return 0;
                    980:                        }
1.1       matt      981:                        if (victim_pvo != NULL)
                    982:                                break;
                    983:                }
                    984:
                    985:                /*
                    986:                 * We also need the pvo entry of the victim we are replacing
                    987:                 * so save the R & C bits of the PTE.
                    988:                 */
                    989:                if ((pt->pte_hi & PTE_HID) == 0 && victim_pvo == NULL &&
                    990:                    pmap_pte_compare(pt, &pvo->pvo_pte)) {
1.15      dyoung    991:                        vpvoh = pvoh;                   /* *1* */
1.1       matt      992:                        victim_pvo = pvo;
                    993:                        if (source_pvo != NULL)
                    994:                                break;
                    995:                }
                    996:        }
                    997:
                    998:        if (source_pvo == NULL) {
                    999:                PMAPCOUNT(ptes_unspilled);
1.50      ad       1000:                PMAP_UNLOCK();
1.1       matt     1001:                return 0;
                   1002:        }
                   1003:
                   1004:        if (victim_pvo == NULL) {
                   1005:                if ((pt->pte_hi & PTE_HID) == 0)
                   1006:                        panic("pmap_pte_spill: victim p-pte (%p) has "
                   1007:                            "no pvo entry!", pt);
                   1008:
                   1009:                /*
                   1010:                 * If this is a secondary PTE, we need to search
                   1011:                 * its primary pvo bucket for the matching PVO.
                   1012:                 */
1.15      dyoung   1013:                vpvoh = &pmap_pvo_table[ptegidx ^ pmap_pteg_mask]; /* *2* */
1.1       matt     1014:                TAILQ_FOREACH(pvo, vpvoh, pvo_olink) {
                   1015:                        PMAP_PVO_CHECK(pvo);            /* sanity check */
                   1016:
                   1017:                        /*
                   1018:                         * We also need the pvo entry of the victim we are
                   1019:                         * replacing so save the R & C bits of the PTE.
                   1020:                         */
                   1021:                        if (pmap_pte_compare(pt, &pvo->pvo_pte)) {
                   1022:                                victim_pvo = pvo;
                   1023:                                break;
                   1024:                        }
                   1025:                }
                   1026:                if (victim_pvo == NULL)
                   1027:                        panic("pmap_pte_spill: victim s-pte (%p) has "
                   1028:                            "no pvo entry!", pt);
                   1029:        }
                   1030:
                   1031:        /*
1.12      matt     1032:         * The victim should be not be a kernel PVO/PTE entry.
                   1033:         */
                   1034:        KASSERT(victim_pvo->pvo_pmap != pmap_kernel());
                   1035:        KASSERT(PVO_PTEGIDX_ISSET(victim_pvo));
                   1036:        KASSERT(PVO_PTEGIDX_GET(victim_pvo) == i);
                   1037:
                   1038:        /*
1.1       matt     1039:         * We are invalidating the TLB entry for the EA for the
                   1040:         * we are replacing even though its valid; If we don't
                   1041:         * we lose any ref/chg bit changes contained in the TLB
                   1042:         * entry.
                   1043:         */
                   1044:        source_pvo->pvo_pte.pte_hi &= ~PTE_HID;
                   1045:
                   1046:        /*
                   1047:         * To enforce the PVO list ordering constraint that all
                   1048:         * evicted entries should come before all valid entries,
                   1049:         * move the source PVO to the tail of its list and the
                   1050:         * victim PVO to the head of its list (which might not be
                   1051:         * the same list, if the victim was using the secondary hash).
                   1052:         */
                   1053:        TAILQ_REMOVE(pvoh, source_pvo, pvo_olink);
                   1054:        TAILQ_INSERT_TAIL(pvoh, source_pvo, pvo_olink);
                   1055:        TAILQ_REMOVE(vpvoh, victim_pvo, pvo_olink);
                   1056:        TAILQ_INSERT_HEAD(vpvoh, victim_pvo, pvo_olink);
                   1057:        pmap_pte_unset(pt, &victim_pvo->pvo_pte, victim_pvo->pvo_vaddr);
                   1058:        pmap_pte_set(pt, &source_pvo->pvo_pte);
                   1059:        victim_pvo->pvo_pmap->pm_evictions++;
                   1060:        source_pvo->pvo_pmap->pm_evictions--;
1.12      matt     1061:        PVO_WHERE(victim_pvo, SPILL_UNSET);
                   1062:        PVO_WHERE(source_pvo, SPILL_SET);
1.1       matt     1063:
                   1064:        PVO_PTEGIDX_CLR(victim_pvo);
                   1065:        PVO_PTEGIDX_SET(source_pvo, i);
                   1066:        PMAPCOUNT2(pmap_evcnt_ptes_primary[i]);
                   1067:        PMAPCOUNT(ptes_spilled);
                   1068:        PMAPCOUNT(ptes_evicted);
                   1069:        PMAPCOUNT(ptes_removed);
                   1070:
                   1071:        PMAP_PVO_CHECK(victim_pvo);
                   1072:        PMAP_PVO_CHECK(source_pvo);
1.50      ad       1073:
                   1074:        PMAP_UNLOCK();
1.1       matt     1075:        return 1;
                   1076: }
                   1077:
                   1078: /*
                   1079:  * Restrict given range to physical memory
                   1080:  */
                   1081: void
                   1082: pmap_real_memory(paddr_t *start, psize_t *size)
                   1083: {
                   1084:        struct mem_region *mp;
                   1085:
                   1086:        for (mp = mem; mp->size; mp++) {
                   1087:                if (*start + *size > mp->start
                   1088:                    && *start < mp->start + mp->size) {
                   1089:                        if (*start < mp->start) {
                   1090:                                *size -= mp->start - *start;
                   1091:                                *start = mp->start;
                   1092:                        }
                   1093:                        if (*start + *size > mp->start + mp->size)
                   1094:                                *size = mp->start + mp->size - *start;
                   1095:                        return;
                   1096:                }
                   1097:        }
                   1098:        *size = 0;
                   1099: }
                   1100:
                   1101: /*
                   1102:  * Initialize anything else for pmap handling.
                   1103:  * Called during vm_init().
                   1104:  */
                   1105: void
                   1106: pmap_init(void)
                   1107: {
                   1108:
                   1109:        pmap_initialized = 1;
                   1110: }
                   1111:
                   1112: /*
1.10      thorpej  1113:  * How much virtual space does the kernel get?
                   1114:  */
                   1115: void
                   1116: pmap_virtual_space(vaddr_t *start, vaddr_t *end)
                   1117: {
                   1118:        /*
                   1119:         * For now, reserve one segment (minus some overhead) for kernel
                   1120:         * virtual memory
                   1121:         */
                   1122:        *start = VM_MIN_KERNEL_ADDRESS;
                   1123:        *end = VM_MAX_KERNEL_ADDRESS;
                   1124: }
                   1125:
                   1126: /*
1.1       matt     1127:  * Allocate, initialize, and return a new physical map.
                   1128:  */
                   1129: pmap_t
                   1130: pmap_create(void)
                   1131: {
                   1132:        pmap_t pm;
1.38      sanjayl  1133:
1.1       matt     1134:        pm = pool_get(&pmap_pool, PR_WAITOK);
1.84      matt     1135:        KASSERT((vaddr_t)pm < VM_MIN_KERNEL_ADDRESS);
1.46      christos 1136:        memset((void *)pm, 0, sizeof *pm);
1.1       matt     1137:        pmap_pinit(pm);
                   1138:
1.85      matt     1139:        DPRINTFN(CREATE, "pmap_create: pm %p:\n"
1.54      mlelstv  1140:            "\t%#" _PRIsr " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr
                   1141:            "    %#" _PRIsr " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr "\n"
                   1142:            "\t%#" _PRIsr " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr
                   1143:            "    %#" _PRIsr " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr "\n",
                   1144:            pm,
                   1145:            pm->pm_sr[0], pm->pm_sr[1],
                   1146:            pm->pm_sr[2], pm->pm_sr[3],
                   1147:            pm->pm_sr[4], pm->pm_sr[5],
                   1148:            pm->pm_sr[6], pm->pm_sr[7],
                   1149:            pm->pm_sr[8], pm->pm_sr[9],
                   1150:            pm->pm_sr[10], pm->pm_sr[11],
                   1151:            pm->pm_sr[12], pm->pm_sr[13],
1.85      matt     1152:            pm->pm_sr[14], pm->pm_sr[15]);
1.1       matt     1153:        return pm;
                   1154: }
                   1155:
                   1156: /*
                   1157:  * Initialize a preallocated and zeroed pmap structure.
                   1158:  */
                   1159: void
                   1160: pmap_pinit(pmap_t pm)
                   1161: {
1.2       matt     1162:        register_t entropy = MFTB();
                   1163:        register_t mask;
                   1164:        int i;
1.1       matt     1165:
                   1166:        /*
                   1167:         * Allocate some segment registers for this pmap.
                   1168:         */
                   1169:        pm->pm_refs = 1;
1.50      ad       1170:        PMAP_LOCK();
1.2       matt     1171:        for (i = 0; i < NPMAPS; i += VSID_NBPW) {
                   1172:                static register_t pmap_vsidcontext;
                   1173:                register_t hash;
                   1174:                unsigned int n;
1.1       matt     1175:
                   1176:                /* Create a new value by multiplying by a prime adding in
                   1177:                 * entropy from the timebase register.  This is to make the
                   1178:                 * VSID more random so that the PT Hash function collides
                   1179:                 * less often. (note that the prime causes gcc to do shifts
                   1180:                 * instead of a multiply)
                   1181:                 */
                   1182:                pmap_vsidcontext = (pmap_vsidcontext * 0x1105) + entropy;
                   1183:                hash = pmap_vsidcontext & (NPMAPS - 1);
1.23      aymeric  1184:                if (hash == 0) {                /* 0 is special, avoid it */
                   1185:                        entropy += 0xbadf00d;
1.1       matt     1186:                        continue;
1.23      aymeric  1187:                }
1.1       matt     1188:                n = hash >> 5;
1.2       matt     1189:                mask = 1L << (hash & (VSID_NBPW-1));
                   1190:                hash = pmap_vsidcontext;
1.1       matt     1191:                if (pmap_vsid_bitmap[n] & mask) {       /* collision? */
                   1192:                        /* anything free in this bucket? */
1.2       matt     1193:                        if (~pmap_vsid_bitmap[n] == 0) {
1.23      aymeric  1194:                                entropy = hash ^ (hash >> 16);
1.1       matt     1195:                                continue;
                   1196:                        }
                   1197:                        i = ffs(~pmap_vsid_bitmap[n]) - 1;
1.2       matt     1198:                        mask = 1L << i;
                   1199:                        hash &= ~(VSID_NBPW-1);
1.1       matt     1200:                        hash |= i;
                   1201:                }
1.18      matt     1202:                hash &= PTE_VSID >> PTE_VSID_SHFT;
1.1       matt     1203:                pmap_vsid_bitmap[n] |= mask;
1.18      matt     1204:                pm->pm_vsid = hash;
1.53      garbled  1205: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1       matt     1206:                for (i = 0; i < 16; i++)
1.14      chs      1207:                        pm->pm_sr[i] = VSID_MAKE(i, hash) | SR_PRKEY |
                   1208:                            SR_NOEXEC;
1.18      matt     1209: #endif
1.50      ad       1210:                PMAP_UNLOCK();
1.1       matt     1211:                return;
                   1212:        }
1.50      ad       1213:        PMAP_UNLOCK();
1.1       matt     1214:        panic("pmap_pinit: out of segments");
                   1215: }
                   1216:
                   1217: /*
                   1218:  * Add a reference to the given pmap.
                   1219:  */
                   1220: void
                   1221: pmap_reference(pmap_t pm)
                   1222: {
1.50      ad       1223:        atomic_inc_uint(&pm->pm_refs);
1.1       matt     1224: }
                   1225:
                   1226: /*
                   1227:  * Retire the given pmap from service.
                   1228:  * Should only be called if the map contains no valid mappings.
                   1229:  */
                   1230: void
                   1231: pmap_destroy(pmap_t pm)
                   1232: {
1.50      ad       1233:        if (atomic_dec_uint_nv(&pm->pm_refs) == 0) {
1.1       matt     1234:                pmap_release(pm);
                   1235:                pool_put(&pmap_pool, pm);
                   1236:        }
                   1237: }
                   1238:
                   1239: /*
                   1240:  * Release any resources held by the given physical map.
                   1241:  * Called when a pmap initialized by pmap_pinit is being released.
                   1242:  */
                   1243: void
                   1244: pmap_release(pmap_t pm)
                   1245: {
                   1246:        int idx, mask;
1.39      matt     1247:
                   1248:        KASSERT(pm->pm_stats.resident_count == 0);
                   1249:        KASSERT(pm->pm_stats.wired_count == 0);
1.1       matt     1250:
1.50      ad       1251:        PMAP_LOCK();
1.1       matt     1252:        if (pm->pm_sr[0] == 0)
                   1253:                panic("pmap_release");
1.22      aymeric  1254:        idx = pm->pm_vsid & (NPMAPS-1);
1.1       matt     1255:        mask = 1 << (idx % VSID_NBPW);
                   1256:        idx /= VSID_NBPW;
1.22      aymeric  1257:
                   1258:        KASSERT(pmap_vsid_bitmap[idx] & mask);
1.1       matt     1259:        pmap_vsid_bitmap[idx] &= ~mask;
1.50      ad       1260:        PMAP_UNLOCK();
1.1       matt     1261: }
                   1262:
                   1263: /*
                   1264:  * Copy the range specified by src_addr/len
                   1265:  * from the source map to the range dst_addr/len
                   1266:  * in the destination map.
                   1267:  *
                   1268:  * This routine is only advisory and need not do anything.
                   1269:  */
                   1270: void
                   1271: pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr,
                   1272:        vsize_t len, vaddr_t src_addr)
                   1273: {
                   1274:        PMAPCOUNT(copies);
                   1275: }
                   1276:
                   1277: /*
                   1278:  * Require that all active physical maps contain no
                   1279:  * incorrect entries NOW.
                   1280:  */
                   1281: void
                   1282: pmap_update(struct pmap *pmap)
                   1283: {
                   1284:        PMAPCOUNT(updates);
                   1285:        TLBSYNC();
                   1286: }
                   1287:
1.35      perry    1288: static inline int
1.1       matt     1289: pmap_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx)
                   1290: {
                   1291:        int pteidx;
                   1292:        /*
                   1293:         * We can find the actual pte entry without searching by
                   1294:         * grabbing the PTEG index from 3 unused bits in pte_lo[11:9]
                   1295:         * and by noticing the HID bit.
                   1296:         */
                   1297:        pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo);
                   1298:        if (pvo->pvo_pte.pte_hi & PTE_HID)
                   1299:                pteidx ^= pmap_pteg_mask * 8;
                   1300:        return pteidx;
                   1301: }
                   1302:
1.2       matt     1303: volatile struct pte *
1.1       matt     1304: pmap_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
                   1305: {
1.2       matt     1306:        volatile struct pte *pt;
1.1       matt     1307:
                   1308: #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK)
                   1309:        if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0)
                   1310:                return NULL;
                   1311: #endif
                   1312:
                   1313:        /*
                   1314:         * If we haven't been supplied the ptegidx, calculate it.
                   1315:         */
                   1316:        if (pteidx == -1) {
                   1317:                int ptegidx;
1.2       matt     1318:                ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr);
1.1       matt     1319:                pteidx = pmap_pvo_pte_index(pvo, ptegidx);
                   1320:        }
                   1321:
                   1322:        pt = &pmap_pteg_table[pteidx >> 3].pt[pteidx & 7];
                   1323:
                   1324: #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK)
                   1325:        return pt;
                   1326: #else
                   1327:        if ((pvo->pvo_pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) {
                   1328:                panic("pmap_pvo_to_pte: pvo %p: has valid pte in "
                   1329:                    "pvo but no valid pte index", pvo);
                   1330:        }
                   1331:        if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) {
                   1332:                panic("pmap_pvo_to_pte: pvo %p: has valid pte index in "
                   1333:                    "pvo but no valid pte", pvo);
                   1334:        }
                   1335:
                   1336:        if ((pt->pte_hi ^ (pvo->pvo_pte.pte_hi & ~PTE_VALID)) == PTE_VALID) {
                   1337:                if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) {
                   1338: #if defined(DEBUG) || defined(PMAPCHECK)
                   1339:                        pmap_pte_print(pt);
                   1340: #endif
                   1341:                        panic("pmap_pvo_to_pte: pvo %p: has valid pte in "
                   1342:                            "pmap_pteg_table %p but invalid in pvo",
                   1343:                            pvo, pt);
                   1344:                }
                   1345:                if (((pt->pte_lo ^ pvo->pvo_pte.pte_lo) & ~(PTE_CHG|PTE_REF)) != 0) {
                   1346: #if defined(DEBUG) || defined(PMAPCHECK)
                   1347:                        pmap_pte_print(pt);
                   1348: #endif
                   1349:                        panic("pmap_pvo_to_pte: pvo %p: pvo pte does "
                   1350:                            "not match pte %p in pmap_pteg_table",
                   1351:                            pvo, pt);
                   1352:                }
                   1353:                return pt;
                   1354:        }
                   1355:
                   1356:        if (pvo->pvo_pte.pte_hi & PTE_VALID) {
                   1357: #if defined(DEBUG) || defined(PMAPCHECK)
                   1358:                pmap_pte_print(pt);
                   1359: #endif
1.12      matt     1360:                panic("pmap_pvo_to_pte: pvo %p: has nomatching pte %p in "
1.1       matt     1361:                    "pmap_pteg_table but valid in pvo", pvo, pt);
                   1362:        }
                   1363:        return NULL;
                   1364: #endif /* !(!DIAGNOSTIC && !DEBUG && !PMAPCHECK) */
                   1365: }
                   1366:
                   1367: struct pvo_entry *
                   1368: pmap_pvo_find_va(pmap_t pm, vaddr_t va, int *pteidx_p)
                   1369: {
                   1370:        struct pvo_entry *pvo;
                   1371:        int ptegidx;
                   1372:
                   1373:        va &= ~ADDR_POFF;
1.2       matt     1374:        ptegidx = va_to_pteg(pm, va);
1.1       matt     1375:
                   1376:        TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
                   1377: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
                   1378:                if ((uintptr_t) pvo >= SEGMENT_LENGTH)
                   1379:                        panic("pmap_pvo_find_va: invalid pvo %p on "
                   1380:                            "list %#x (%p)", pvo, ptegidx,
                   1381:                             &pmap_pvo_table[ptegidx]);
                   1382: #endif
                   1383:                if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
                   1384:                        if (pteidx_p)
                   1385:                                *pteidx_p = pmap_pvo_pte_index(pvo, ptegidx);
                   1386:                        return pvo;
                   1387:                }
                   1388:        }
1.38      sanjayl  1389:        if ((pm == pmap_kernel()) && (va < SEGMENT_LENGTH))
1.54      mlelstv  1390:                panic("%s: returning NULL for %s pmap, va: %#" _PRIxva "\n",
1.53      garbled  1391:                    __func__, (pm == pmap_kernel() ? "kernel" : "user"), va);
1.1       matt     1392:        return NULL;
                   1393: }
                   1394:
                   1395: #if defined(DEBUG) || defined(PMAPCHECK)
                   1396: void
                   1397: pmap_pvo_check(const struct pvo_entry *pvo)
                   1398: {
                   1399:        struct pvo_head *pvo_head;
                   1400:        struct pvo_entry *pvo0;
1.2       matt     1401:        volatile struct pte *pt;
1.1       matt     1402:        int failed = 0;
                   1403:
1.50      ad       1404:        PMAP_LOCK();
                   1405:
1.1       matt     1406:        if ((uintptr_t)(pvo+1) >= SEGMENT_LENGTH)
                   1407:                panic("pmap_pvo_check: pvo %p: invalid address", pvo);
                   1408:
                   1409:        if ((uintptr_t)(pvo->pvo_pmap+1) >= SEGMENT_LENGTH) {
                   1410:                printf("pmap_pvo_check: pvo %p: invalid pmap address %p\n",
                   1411:                    pvo, pvo->pvo_pmap);
                   1412:                failed = 1;
                   1413:        }
                   1414:
                   1415:        if ((uintptr_t)TAILQ_NEXT(pvo, pvo_olink) >= SEGMENT_LENGTH ||
                   1416:            (((uintptr_t)TAILQ_NEXT(pvo, pvo_olink)) & 0x1f) != 0) {
                   1417:                printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n",
                   1418:                    pvo, TAILQ_NEXT(pvo, pvo_olink));
                   1419:                failed = 1;
                   1420:        }
                   1421:
                   1422:        if ((uintptr_t)LIST_NEXT(pvo, pvo_vlink) >= SEGMENT_LENGTH ||
                   1423:            (((uintptr_t)LIST_NEXT(pvo, pvo_vlink)) & 0x1f) != 0) {
                   1424:                printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n",
                   1425:                    pvo, LIST_NEXT(pvo, pvo_vlink));
                   1426:                failed = 1;
                   1427:        }
                   1428:
1.39      matt     1429:        if (PVO_MANAGED_P(pvo)) {
1.1       matt     1430:                pvo_head = pa_to_pvoh(pvo->pvo_pte.pte_lo & PTE_RPGN, NULL);
1.107     chs      1431:                LIST_FOREACH(pvo0, pvo_head, pvo_vlink) {
                   1432:                        if (pvo0 == pvo)
                   1433:                                break;
                   1434:                }
                   1435:                if (pvo0 == NULL) {
                   1436:                        printf("pmap_pvo_check: pvo %p: not present "
                   1437:                               "on its vlist head %p\n", pvo, pvo_head);
                   1438:                        failed = 1;
                   1439:                }
1.1       matt     1440:        } else {
1.107     chs      1441:                KASSERT(pvo->pvo_vaddr >= VM_MIN_KERNEL_ADDRESS);
                   1442:                if (__predict_false(pvo->pvo_vaddr < VM_MIN_KERNEL_ADDRESS))
1.1       matt     1443:                        failed = 1;
                   1444:        }
                   1445:        if (pvo != pmap_pvo_find_va(pvo->pvo_pmap, pvo->pvo_vaddr, NULL)) {
                   1446:                printf("pmap_pvo_check: pvo %p: not present "
                   1447:                    "on its olist head\n", pvo);
                   1448:                failed = 1;
                   1449:        }
                   1450:        pt = pmap_pvo_to_pte(pvo, -1);
                   1451:        if (pt == NULL) {
                   1452:                if (pvo->pvo_pte.pte_hi & PTE_VALID) {
                   1453:                        printf("pmap_pvo_check: pvo %p: pte_hi VALID but "
                   1454:                            "no PTE\n", pvo);
                   1455:                        failed = 1;
                   1456:                }
                   1457:        } else {
                   1458:                if ((uintptr_t) pt < (uintptr_t) &pmap_pteg_table[0] ||
                   1459:                    (uintptr_t) pt >=
                   1460:                    (uintptr_t) &pmap_pteg_table[pmap_pteg_cnt]) {
                   1461:                        printf("pmap_pvo_check: pvo %p: pte %p not in "
                   1462:                            "pteg table\n", pvo, pt);
                   1463:                        failed = 1;
                   1464:                }
                   1465:                if (((((uintptr_t) pt) >> 3) & 7) != PVO_PTEGIDX_GET(pvo)) {
                   1466:                        printf("pmap_pvo_check: pvo %p: pte_hi VALID but "
                   1467:                            "no PTE\n", pvo);
                   1468:                        failed = 1;
                   1469:                }
                   1470:                if (pvo->pvo_pte.pte_hi != pt->pte_hi) {
                   1471:                        printf("pmap_pvo_check: pvo %p: pte_hi differ: "
1.54      mlelstv  1472:                            "%#" _PRIxpte "/%#" _PRIxpte "\n", pvo,
                   1473:                            pvo->pvo_pte.pte_hi,
                   1474:                            pt->pte_hi);
1.1       matt     1475:                        failed = 1;
                   1476:                }
                   1477:                if (((pvo->pvo_pte.pte_lo ^ pt->pte_lo) &
                   1478:                    (PTE_PP|PTE_WIMG|PTE_RPGN)) != 0) {
                   1479:                        printf("pmap_pvo_check: pvo %p: pte_lo differ: "
1.54      mlelstv  1480:                            "%#" _PRIxpte "/%#" _PRIxpte "\n", pvo,
                   1481:                            (pvo->pvo_pte.pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN)),
                   1482:                            (pt->pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN)));
1.1       matt     1483:                        failed = 1;
                   1484:                }
                   1485:                if ((pmap_pte_to_va(pt) ^ PVO_VADDR(pvo)) & 0x0fffffff) {
1.53      garbled  1486:                        printf("pmap_pvo_check: pvo %p: PTE %p derived VA %#" _PRIxva ""
                   1487:                            " doesn't not match PVO's VA %#" _PRIxva "\n",
1.1       matt     1488:                            pvo, pt, pmap_pte_to_va(pt), PVO_VADDR(pvo));
                   1489:                        failed = 1;
                   1490:                }
                   1491:                if (failed)
                   1492:                        pmap_pte_print(pt);
                   1493:        }
                   1494:        if (failed)
                   1495:                panic("pmap_pvo_check: pvo %p, pm %p: bugcheck!", pvo,
                   1496:                    pvo->pvo_pmap);
1.50      ad       1497:
                   1498:        PMAP_UNLOCK();
1.1       matt     1499: }
                   1500: #endif /* DEBUG || PMAPCHECK */
                   1501:
                   1502: /*
1.25      chs      1503:  * Search the PVO table looking for a non-wired entry.
                   1504:  * If we find one, remove it and return it.
                   1505:  */
                   1506:
                   1507: struct pvo_entry *
                   1508: pmap_pvo_reclaim(struct pmap *pm)
                   1509: {
                   1510:        struct pvo_tqhead *pvoh;
                   1511:        struct pvo_entry *pvo;
                   1512:        uint32_t idx, endidx;
                   1513:
                   1514:        endidx = pmap_pvo_reclaim_nextidx;
                   1515:        for (idx = (endidx + 1) & pmap_pteg_mask; idx != endidx;
                   1516:             idx = (idx + 1) & pmap_pteg_mask) {
                   1517:                pvoh = &pmap_pvo_table[idx];
                   1518:                TAILQ_FOREACH(pvo, pvoh, pvo_olink) {
1.39      matt     1519:                        if (!PVO_WIRED_P(pvo)) {
1.33      chs      1520:                                pmap_pvo_remove(pvo, -1, NULL);
1.25      chs      1521:                                pmap_pvo_reclaim_nextidx = idx;
1.26      matt     1522:                                PMAPCOUNT(pvos_reclaimed);
1.25      chs      1523:                                return pvo;
                   1524:                        }
                   1525:                }
                   1526:        }
                   1527:        return NULL;
                   1528: }
                   1529:
                   1530: /*
1.1       matt     1531:  * This returns whether this is the first mapping of a page.
                   1532:  */
                   1533: int
                   1534: pmap_pvo_enter(pmap_t pm, struct pool *pl, struct pvo_head *pvo_head,
1.2       matt     1535:        vaddr_t va, paddr_t pa, register_t pte_lo, int flags)
1.1       matt     1536: {
                   1537:        struct pvo_entry *pvo;
                   1538:        struct pvo_tqhead *pvoh;
1.2       matt     1539:        register_t msr;
1.1       matt     1540:        int ptegidx;
                   1541:        int i;
                   1542:        int poolflags = PR_NOWAIT;
                   1543:
1.28      chs      1544:        /*
                   1545:         * Compute the PTE Group index.
                   1546:         */
                   1547:        va &= ~ADDR_POFF;
                   1548:        ptegidx = va_to_pteg(pm, va);
                   1549:
                   1550:        msr = pmap_interrupts_off();
                   1551:
1.1       matt     1552: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
                   1553:        if (pmap_pvo_remove_depth > 0)
                   1554:                panic("pmap_pvo_enter: called while pmap_pvo_remove active!");
                   1555:        if (++pmap_pvo_enter_depth > 1)
                   1556:                panic("pmap_pvo_enter: called recursively!");
                   1557: #endif
                   1558:
                   1559:        /*
                   1560:         * Remove any existing mapping for this page.  Reuse the
                   1561:         * pvo entry if there a mapping.
                   1562:         */
                   1563:        TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
                   1564:                if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
                   1565: #ifdef DEBUG
                   1566:                        if ((pmapdebug & PMAPDEBUG_PVOENTER) &&
                   1567:                            ((pvo->pvo_pte.pte_lo ^ (pa|pte_lo)) &
                   1568:                            ~(PTE_REF|PTE_CHG)) == 0 &&
                   1569:                           va < VM_MIN_KERNEL_ADDRESS) {
1.56      phx      1570:                                printf("pmap_pvo_enter: pvo %p: dup %#" _PRIxpte "/%#" _PRIxpa "\n",
1.54      mlelstv  1571:                                    pvo, pvo->pvo_pte.pte_lo, pte_lo|pa);
1.56      phx      1572:                                printf("pmap_pvo_enter: pte_hi=%#" _PRIxpte " sr=%#" _PRIsr "\n",
1.54      mlelstv  1573:                                    pvo->pvo_pte.pte_hi,
                   1574:                                    pm->pm_sr[va >> ADDR_SR_SHFT]);
1.1       matt     1575:                                pmap_pte_print(pmap_pvo_to_pte(pvo, -1));
                   1576: #ifdef DDBX
                   1577:                                Debugger();
                   1578: #endif
                   1579:                        }
                   1580: #endif
                   1581:                        PMAPCOUNT(mappings_replaced);
1.33      chs      1582:                        pmap_pvo_remove(pvo, -1, NULL);
1.1       matt     1583:                        break;
                   1584:                }
                   1585:        }
                   1586:
                   1587:        /*
                   1588:         * If we aren't overwriting an mapping, try to allocate
                   1589:         */
1.26      matt     1590: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
                   1591:        --pmap_pvo_enter_depth;
                   1592: #endif
1.1       matt     1593:        pmap_interrupts_restore(msr);
1.106     martin   1594:        if (pvo == NULL) {
1.95      chs      1595:                pvo = pool_get(pl, poolflags);
1.33      chs      1596:        }
1.84      matt     1597:        KASSERT((vaddr_t)pvo < VM_MIN_KERNEL_ADDRESS);
1.25      chs      1598:
                   1599: #ifdef DEBUG
                   1600:        /*
                   1601:         * Exercise pmap_pvo_reclaim() a little.
                   1602:         */
                   1603:        if (pvo && (flags & PMAP_CANFAIL) != 0 &&
                   1604:            pmap_pvo_reclaim_debugctr++ > 0x1000 &&
                   1605:            (pmap_pvo_reclaim_debugctr & 0xff) == 0) {
                   1606:                pool_put(pl, pvo);
                   1607:                pvo = NULL;
                   1608:        }
                   1609: #endif
                   1610:
1.1       matt     1611:        msr = pmap_interrupts_off();
1.26      matt     1612: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
                   1613:        ++pmap_pvo_enter_depth;
                   1614: #endif
1.1       matt     1615:        if (pvo == NULL) {
                   1616:                pvo = pmap_pvo_reclaim(pm);
                   1617:                if (pvo == NULL) {
                   1618:                        if ((flags & PMAP_CANFAIL) == 0)
                   1619:                                panic("pmap_pvo_enter: failed");
                   1620: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
                   1621:                        pmap_pvo_enter_depth--;
                   1622: #endif
1.26      matt     1623:                        PMAPCOUNT(pvos_failed);
1.1       matt     1624:                        pmap_interrupts_restore(msr);
                   1625:                        return ENOMEM;
                   1626:                }
                   1627:        }
1.25      chs      1628:
1.1       matt     1629:        pvo->pvo_vaddr = va;
                   1630:        pvo->pvo_pmap = pm;
                   1631:        pvo->pvo_vaddr &= ~ADDR_POFF;
                   1632:        if (flags & VM_PROT_EXECUTE) {
                   1633:                PMAPCOUNT(exec_mappings);
1.14      chs      1634:                pvo_set_exec(pvo);
1.1       matt     1635:        }
                   1636:        if (flags & PMAP_WIRED)
                   1637:                pvo->pvo_vaddr |= PVO_WIRED;
1.107     chs      1638:        if (pvo_head != NULL) {
1.1       matt     1639:                pvo->pvo_vaddr |= PVO_MANAGED;
                   1640:                PMAPCOUNT(mappings);
                   1641:        } else {
                   1642:                PMAPCOUNT(kernel_mappings);
                   1643:        }
1.2       matt     1644:        pmap_pte_create(&pvo->pvo_pte, pm, va, pa | pte_lo);
1.1       matt     1645:
1.107     chs      1646:        if (pvo_head != NULL)
                   1647:                LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink);
1.39      matt     1648:        if (PVO_WIRED_P(pvo))
1.1       matt     1649:                pvo->pvo_pmap->pm_stats.wired_count++;
                   1650:        pvo->pvo_pmap->pm_stats.resident_count++;
                   1651: #if defined(DEBUG)
1.38      sanjayl  1652: /*     if (pm != pmap_kernel() && va < VM_MIN_KERNEL_ADDRESS) */
1.1       matt     1653:                DPRINTFN(PVOENTER,
1.85      matt     1654:                    "pmap_pvo_enter: pvo %p: pm %p va %#" _PRIxva " pa %#" _PRIxpa "\n",
                   1655:                    pvo, pm, va, pa);
1.1       matt     1656: #endif
                   1657:
                   1658:        /*
                   1659:         * We hope this succeeds but it isn't required.
                   1660:         */
                   1661:        pvoh = &pmap_pvo_table[ptegidx];
                   1662:        i = pmap_pte_insert(ptegidx, &pvo->pvo_pte);
                   1663:        if (i >= 0) {
                   1664:                PVO_PTEGIDX_SET(pvo, i);
1.12      matt     1665:                PVO_WHERE(pvo, ENTER_INSERT);
1.1       matt     1666:                PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID)
                   1667:                    ? pmap_evcnt_ptes_secondary : pmap_evcnt_ptes_primary)[i]);
                   1668:                TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink);
1.38      sanjayl  1669:
1.1       matt     1670:        } else {
                   1671:                /*
                   1672:                 * Since we didn't have room for this entry (which makes it
                   1673:                 * and evicted entry), place it at the head of the list.
                   1674:                 */
                   1675:                TAILQ_INSERT_HEAD(pvoh, pvo, pvo_olink);
                   1676:                PMAPCOUNT(ptes_evicted);
                   1677:                pm->pm_evictions++;
1.12      matt     1678:                /*
                   1679:                 * If this is a kernel page, make sure it's active.
                   1680:                 */
                   1681:                if (pm == pmap_kernel()) {
1.45      thorpej  1682:                        i = pmap_pte_spill(pm, va, false);
1.12      matt     1683:                        KASSERT(i);
                   1684:                }
1.1       matt     1685:        }
                   1686:        PMAP_PVO_CHECK(pvo);            /* sanity check */
                   1687: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
                   1688:        pmap_pvo_enter_depth--;
                   1689: #endif
                   1690:        pmap_interrupts_restore(msr);
                   1691:        return 0;
                   1692: }
                   1693:
1.53      garbled  1694: static void
1.33      chs      1695: pmap_pvo_remove(struct pvo_entry *pvo, int pteidx, struct pvo_head *pvol)
1.1       matt     1696: {
1.2       matt     1697:        volatile struct pte *pt;
1.1       matt     1698:        int ptegidx;
                   1699:
                   1700: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
                   1701:        if (++pmap_pvo_remove_depth > 1)
                   1702:                panic("pmap_pvo_remove: called recursively!");
                   1703: #endif
                   1704:
                   1705:        /*
                   1706:         * If we haven't been supplied the ptegidx, calculate it.
                   1707:         */
                   1708:        if (pteidx == -1) {
1.2       matt     1709:                ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr);
1.1       matt     1710:                pteidx = pmap_pvo_pte_index(pvo, ptegidx);
                   1711:        } else {
                   1712:                ptegidx = pteidx >> 3;
                   1713:                if (pvo->pvo_pte.pte_hi & PTE_HID)
                   1714:                        ptegidx ^= pmap_pteg_mask;
                   1715:        }
                   1716:        PMAP_PVO_CHECK(pvo);            /* sanity check */
                   1717:
                   1718:        /*
                   1719:         * If there is an active pte entry, we need to deactivate it
                   1720:         * (and save the ref & chg bits).
                   1721:         */
                   1722:        pt = pmap_pvo_to_pte(pvo, pteidx);
                   1723:        if (pt != NULL) {
                   1724:                pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
1.12      matt     1725:                PVO_WHERE(pvo, REMOVE);
1.1       matt     1726:                PVO_PTEGIDX_CLR(pvo);
                   1727:                PMAPCOUNT(ptes_removed);
                   1728:        } else {
                   1729:                KASSERT(pvo->pvo_pmap->pm_evictions > 0);
                   1730:                pvo->pvo_pmap->pm_evictions--;
                   1731:        }
                   1732:
                   1733:        /*
1.14      chs      1734:         * Account for executable mappings.
                   1735:         */
1.39      matt     1736:        if (PVO_EXECUTABLE_P(pvo))
1.14      chs      1737:                pvo_clear_exec(pvo);
                   1738:
                   1739:        /*
                   1740:         * Update our statistics.
1.1       matt     1741:         */
                   1742:        pvo->pvo_pmap->pm_stats.resident_count--;
1.39      matt     1743:        if (PVO_WIRED_P(pvo))
1.1       matt     1744:                pvo->pvo_pmap->pm_stats.wired_count--;
                   1745:
                   1746:        /*
1.107     chs      1747:         * If the page is managed:
                   1748:         * Save the REF/CHG bits into their cache.
                   1749:         * Remove the PVO from the P/V list.
1.1       matt     1750:         */
1.39      matt     1751:        if (PVO_MANAGED_P(pvo)) {
1.2       matt     1752:                register_t ptelo = pvo->pvo_pte.pte_lo;
1.1       matt     1753:                struct vm_page *pg = PHYS_TO_VM_PAGE(ptelo & PTE_RPGN);
                   1754:
                   1755:                if (pg != NULL) {
1.37      matt     1756:                        /*
                   1757:                         * If this page was changed and it is mapped exec,
                   1758:                         * invalidate it.
                   1759:                         */
                   1760:                        if ((ptelo & PTE_CHG) &&
                   1761:                            (pmap_attr_fetch(pg) & PTE_EXEC)) {
                   1762:                                struct pvo_head *pvoh = vm_page_to_pvoh(pg);
                   1763:                                if (LIST_EMPTY(pvoh)) {
1.85      matt     1764:                                        DPRINTFN(EXEC, "[pmap_pvo_remove: "
1.53      garbled  1765:                                            "%#" _PRIxpa ": clear-exec]\n",
1.85      matt     1766:                                            VM_PAGE_TO_PHYS(pg));
1.37      matt     1767:                                        pmap_attr_clear(pg, PTE_EXEC);
                   1768:                                        PMAPCOUNT(exec_uncached_pvo_remove);
                   1769:                                } else {
1.85      matt     1770:                                        DPRINTFN(EXEC, "[pmap_pvo_remove: "
1.53      garbled  1771:                                            "%#" _PRIxpa ": syncicache]\n",
1.85      matt     1772:                                            VM_PAGE_TO_PHYS(pg));
1.37      matt     1773:                                        pmap_syncicache(VM_PAGE_TO_PHYS(pg),
                   1774:                                            PAGE_SIZE);
                   1775:                                        PMAPCOUNT(exec_synced_pvo_remove);
                   1776:                                }
                   1777:                        }
                   1778:
1.1       matt     1779:                        pmap_attr_save(pg, ptelo & (PTE_REF|PTE_CHG));
                   1780:                }
1.107     chs      1781:                LIST_REMOVE(pvo, pvo_vlink);
1.1       matt     1782:                PMAPCOUNT(unmappings);
                   1783:        } else {
                   1784:                PMAPCOUNT(kernel_unmappings);
                   1785:        }
                   1786:
                   1787:        /*
1.107     chs      1788:         * Remove the PVO from its list and return it to the pool.
1.1       matt     1789:         */
                   1790:        TAILQ_REMOVE(&pmap_pvo_table[ptegidx], pvo, pvo_olink);
1.33      chs      1791:        if (pvol) {
                   1792:                LIST_INSERT_HEAD(pvol, pvo, pvo_vlink);
1.25      chs      1793:        }
1.1       matt     1794: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
                   1795:        pmap_pvo_remove_depth--;
                   1796: #endif
                   1797: }
                   1798:
1.33      chs      1799: void
                   1800: pmap_pvo_free(struct pvo_entry *pvo)
                   1801: {
                   1802:
1.106     martin   1803:        pool_put(&pmap_pvo_pool, pvo);
1.33      chs      1804: }
                   1805:
                   1806: void
                   1807: pmap_pvo_free_list(struct pvo_head *pvol)
                   1808: {
                   1809:        struct pvo_entry *pvo, *npvo;
                   1810:
                   1811:        for (pvo = LIST_FIRST(pvol); pvo != NULL; pvo = npvo) {
                   1812:                npvo = LIST_NEXT(pvo, pvo_vlink);
                   1813:                LIST_REMOVE(pvo, pvo_vlink);
                   1814:                pmap_pvo_free(pvo);
                   1815:        }
                   1816: }
                   1817:
1.1       matt     1818: /*
1.14      chs      1819:  * Mark a mapping as executable.
                   1820:  * If this is the first executable mapping in the segment,
                   1821:  * clear the noexec flag.
                   1822:  */
1.53      garbled  1823: static void
1.14      chs      1824: pvo_set_exec(struct pvo_entry *pvo)
                   1825: {
                   1826:        struct pmap *pm = pvo->pvo_pmap;
                   1827:
1.39      matt     1828:        if (pm == pmap_kernel() || PVO_EXECUTABLE_P(pvo)) {
1.14      chs      1829:                return;
                   1830:        }
                   1831:        pvo->pvo_vaddr |= PVO_EXECUTABLE;
1.53      garbled  1832: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.18      matt     1833:        {
                   1834:                int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT;
                   1835:                if (pm->pm_exec[sr]++ == 0) {
                   1836:                        pm->pm_sr[sr] &= ~SR_NOEXEC;
                   1837:                }
1.14      chs      1838:        }
1.18      matt     1839: #endif
1.14      chs      1840: }
                   1841:
                   1842: /*
                   1843:  * Mark a mapping as non-executable.
                   1844:  * If this was the last executable mapping in the segment,
                   1845:  * set the noexec flag.
                   1846:  */
1.53      garbled  1847: static void
1.14      chs      1848: pvo_clear_exec(struct pvo_entry *pvo)
                   1849: {
                   1850:        struct pmap *pm = pvo->pvo_pmap;
                   1851:
1.39      matt     1852:        if (pm == pmap_kernel() || !PVO_EXECUTABLE_P(pvo)) {
1.14      chs      1853:                return;
                   1854:        }
                   1855:        pvo->pvo_vaddr &= ~PVO_EXECUTABLE;
1.53      garbled  1856: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.18      matt     1857:        {
                   1858:                int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT;
                   1859:                if (--pm->pm_exec[sr] == 0) {
                   1860:                        pm->pm_sr[sr] |= SR_NOEXEC;
                   1861:                }
1.14      chs      1862:        }
1.18      matt     1863: #endif
1.14      chs      1864: }
                   1865:
                   1866: /*
1.1       matt     1867:  * Insert physical page at pa into the given pmap at virtual address va.
                   1868:  */
                   1869: int
1.65      cegger   1870: pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1.1       matt     1871: {
                   1872:        struct mem_region *mp;
                   1873:        struct pvo_head *pvo_head;
                   1874:        struct vm_page *pg;
1.2       matt     1875:        register_t pte_lo;
1.1       matt     1876:        int error;
                   1877:        u_int was_exec = 0;
                   1878:
1.50      ad       1879:        PMAP_LOCK();
                   1880:
1.1       matt     1881:        if (__predict_false(!pmap_initialized)) {
1.107     chs      1882:                pvo_head = NULL;
1.1       matt     1883:                pg = NULL;
                   1884:                was_exec = PTE_EXEC;
1.107     chs      1885:
1.1       matt     1886:        } else {
                   1887:                pvo_head = pa_to_pvoh(pa, &pg);
                   1888:        }
                   1889:
                   1890:        DPRINTFN(ENTER,
1.85      matt     1891:            "pmap_enter(%p, %#" _PRIxva ", %#" _PRIxpa ", 0x%x, 0x%x):",
                   1892:            pm, va, pa, prot, flags);
1.1       matt     1893:
                   1894:        /*
                   1895:         * If this is a managed page, and it's the first reference to the
                   1896:         * page clear the execness of the page.  Otherwise fetch the execness.
                   1897:         */
                   1898:        if (pg != NULL)
                   1899:                was_exec = pmap_attr_fetch(pg) & PTE_EXEC;
                   1900:
1.85      matt     1901:        DPRINTFN(ENTER, " was_exec=%d", was_exec);
1.1       matt     1902:
                   1903:        /*
                   1904:         * Assume the page is cache inhibited and access is guarded unless
                   1905:         * it's in our available memory array.  If it is in the memory array,
                   1906:         * asssume it's in memory coherent memory.
                   1907:         */
1.77      macallan 1908:        if (flags & PMAP_MD_PREFETCHABLE) {
                   1909:                pte_lo = 0;
                   1910:        } else
                   1911:                pte_lo = PTE_G;
                   1912:
1.81      matt     1913:        if ((flags & PMAP_NOCACHE) == 0) {
1.1       matt     1914:                for (mp = mem; mp->size; mp++) {
                   1915:                        if (pa >= mp->start && pa < mp->start + mp->size) {
                   1916:                                pte_lo = PTE_M;
                   1917:                                break;
                   1918:                        }
                   1919:                }
1.87      kiyohara 1920: #ifdef MULTIPROCESSOR
                   1921:                if (((mfpvr() >> 16) & 0xffff) == MPC603e)
                   1922:                        pte_lo = PTE_M;
                   1923: #endif
1.77      macallan 1924:        } else {
                   1925:                pte_lo |= PTE_I;
1.1       matt     1926:        }
                   1927:
                   1928:        if (prot & VM_PROT_WRITE)
                   1929:                pte_lo |= PTE_BW;
                   1930:        else
                   1931:                pte_lo |= PTE_BR;
                   1932:
                   1933:        /*
                   1934:         * If this was in response to a fault, "pre-fault" the PTE's
                   1935:         * changed/referenced bit appropriately.
                   1936:         */
                   1937:        if (flags & VM_PROT_WRITE)
                   1938:                pte_lo |= PTE_CHG;
1.30      chs      1939:        if (flags & VM_PROT_ALL)
1.1       matt     1940:                pte_lo |= PTE_REF;
                   1941:
                   1942:        /*
                   1943:         * We need to know if this page can be executable
                   1944:         */
                   1945:        flags |= (prot & VM_PROT_EXECUTE);
                   1946:
                   1947:        /*
                   1948:         * Record mapping for later back-translation and pte spilling.
                   1949:         * This will overwrite any existing mapping.
                   1950:         */
1.106     martin   1951:        error = pmap_pvo_enter(pm, &pmap_pvo_pool, pvo_head, va, pa, pte_lo, flags);
1.1       matt     1952:
                   1953:        /*
                   1954:         * Flush the real page from the instruction cache if this page is
                   1955:         * mapped executable and cacheable and has not been flushed since
                   1956:         * the last time it was modified.
                   1957:         */
                   1958:        if (error == 0 &&
                   1959:             (flags & VM_PROT_EXECUTE) &&
                   1960:             (pte_lo & PTE_I) == 0 &&
                   1961:            was_exec == 0) {
1.85      matt     1962:                DPRINTFN(ENTER, " %s", "syncicache");
1.1       matt     1963:                PMAPCOUNT(exec_synced);
1.6       thorpej  1964:                pmap_syncicache(pa, PAGE_SIZE);
1.1       matt     1965:                if (pg != NULL) {
                   1966:                        pmap_attr_save(pg, PTE_EXEC);
                   1967:                        PMAPCOUNT(exec_cached);
                   1968: #if defined(DEBUG) || defined(PMAPDEBUG)
                   1969:                        if (pmapdebug & PMAPDEBUG_ENTER)
                   1970:                                printf(" marked-as-exec");
                   1971:                        else if (pmapdebug & PMAPDEBUG_EXEC)
1.53      garbled  1972:                                printf("[pmap_enter: %#" _PRIxpa ": marked-as-exec]\n",
1.34      yamt     1973:                                    VM_PAGE_TO_PHYS(pg));
1.1       matt     1974: #endif
                   1975:                }
                   1976:        }
                   1977:
1.85      matt     1978:        DPRINTFN(ENTER, ": error=%d\n", error);
1.1       matt     1979:
1.50      ad       1980:        PMAP_UNLOCK();
                   1981:
1.1       matt     1982:        return error;
                   1983: }
                   1984:
                   1985: void
1.68      cegger   1986: pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1.1       matt     1987: {
                   1988:        struct mem_region *mp;
1.2       matt     1989:        register_t pte_lo;
1.1       matt     1990:        int error;
                   1991:
1.85      matt     1992: #if defined (PMAP_OEA64_BRIDGE) || defined (PMAP_OEA)
1.1       matt     1993:        if (va < VM_MIN_KERNEL_ADDRESS)
                   1994:                panic("pmap_kenter_pa: attempt to enter "
1.53      garbled  1995:                    "non-kernel address %#" _PRIxva "!", va);
1.38      sanjayl  1996: #endif
1.1       matt     1997:
                   1998:        DPRINTFN(KENTER,
1.85      matt     1999:            "pmap_kenter_pa(%#" _PRIxva ",%#" _PRIxpa ",%#x)\n", va, pa, prot);
1.1       matt     2000:
1.50      ad       2001:        PMAP_LOCK();
                   2002:
1.1       matt     2003:        /*
                   2004:         * Assume the page is cache inhibited and access is guarded unless
                   2005:         * it's in our available memory array.  If it is in the memory array,
                   2006:         * asssume it's in memory coherent memory.
                   2007:         */
                   2008:        pte_lo = PTE_IG;
1.81      matt     2009:        if ((flags & PMAP_NOCACHE) == 0) {
1.4       matt     2010:                for (mp = mem; mp->size; mp++) {
                   2011:                        if (pa >= mp->start && pa < mp->start + mp->size) {
                   2012:                                pte_lo = PTE_M;
                   2013:                                break;
                   2014:                        }
1.1       matt     2015:                }
1.87      kiyohara 2016: #ifdef MULTIPROCESSOR
                   2017:                if (((mfpvr() >> 16) & 0xffff) == MPC603e)
                   2018:                        pte_lo = PTE_M;
                   2019: #endif
1.1       matt     2020:        }
                   2021:
                   2022:        if (prot & VM_PROT_WRITE)
                   2023:                pte_lo |= PTE_BW;
                   2024:        else
                   2025:                pte_lo |= PTE_BR;
                   2026:
                   2027:        /*
                   2028:         * We don't care about REF/CHG on PVOs on the unmanaged list.
                   2029:         */
1.106     martin   2030:        error = pmap_pvo_enter(pmap_kernel(), &pmap_pvo_pool,
1.107     chs      2031:            NULL, va, pa, pte_lo, prot|PMAP_WIRED);
1.1       matt     2032:
                   2033:        if (error != 0)
1.53      garbled  2034:                panic("pmap_kenter_pa: failed to enter va %#" _PRIxva " pa %#" _PRIxpa ": %d",
1.1       matt     2035:                      va, pa, error);
1.50      ad       2036:
                   2037:        PMAP_UNLOCK();
1.1       matt     2038: }
                   2039:
                   2040: void
                   2041: pmap_kremove(vaddr_t va, vsize_t len)
                   2042: {
                   2043:        if (va < VM_MIN_KERNEL_ADDRESS)
                   2044:                panic("pmap_kremove: attempt to remove "
1.53      garbled  2045:                    "non-kernel address %#" _PRIxva "!", va);
1.1       matt     2046:
1.85      matt     2047:        DPRINTFN(KREMOVE, "pmap_kremove(%#" _PRIxva ",%#" _PRIxva ")\n", va, len);
1.1       matt     2048:        pmap_remove(pmap_kernel(), va, va + len);
                   2049: }
                   2050:
                   2051: /*
                   2052:  * Remove the given range of mapping entries.
                   2053:  */
                   2054: void
                   2055: pmap_remove(pmap_t pm, vaddr_t va, vaddr_t endva)
                   2056: {
1.33      chs      2057:        struct pvo_head pvol;
1.1       matt     2058:        struct pvo_entry *pvo;
1.2       matt     2059:        register_t msr;
1.1       matt     2060:        int pteidx;
                   2061:
1.50      ad       2062:        PMAP_LOCK();
1.33      chs      2063:        LIST_INIT(&pvol);
1.14      chs      2064:        msr = pmap_interrupts_off();
1.1       matt     2065:        for (; va < endva; va += PAGE_SIZE) {
                   2066:                pvo = pmap_pvo_find_va(pm, va, &pteidx);
                   2067:                if (pvo != NULL) {
1.33      chs      2068:                        pmap_pvo_remove(pvo, pteidx, &pvol);
1.1       matt     2069:                }
                   2070:        }
1.14      chs      2071:        pmap_interrupts_restore(msr);
1.33      chs      2072:        pmap_pvo_free_list(&pvol);
1.50      ad       2073:        PMAP_UNLOCK();
1.1       matt     2074: }
                   2075:
1.104     thorpej  2076: #if defined(PMAP_OEA)
                   2077: #ifdef PPC_OEA601
                   2078: bool
                   2079: pmap_extract_ioseg601(vaddr_t va, paddr_t *pap)
                   2080: {
                   2081:        if ((MFPVR() >> 16) != MPC601)
                   2082:                return false;
                   2083:
                   2084:        const register_t sr = iosrtable[va >> ADDR_SR_SHFT];
                   2085:
                   2086:        if (SR601_VALID_P(sr) && SR601_PA_MATCH_P(sr, va)) {
                   2087:                if (pap)
                   2088:                        *pap = va;
                   2089:                return true;
                   2090:        }
                   2091:        return false;
                   2092: }
                   2093:
                   2094: static bool
                   2095: pmap_extract_battable601(vaddr_t va, paddr_t *pap)
                   2096: {
                   2097:        const register_t batu = battable[va >> 23].batu;
                   2098:        const register_t batl = battable[va >> 23].batl;
                   2099:
                   2100:        if (BAT601_VALID_P(batl) && BAT601_VA_MATCH_P(batu, batl, va)) {
                   2101:                const register_t mask =
                   2102:                    (~(batl & BAT601_BSM) << 17) & ~0x1ffffL;
                   2103:                if (pap)
                   2104:                        *pap = (batl & mask) | (va & ~mask);
                   2105:                return true;
                   2106:        }
                   2107:        return false;
                   2108: }
                   2109: #endif /* PPC_OEA601 */
                   2110:
                   2111: bool
                   2112: pmap_extract_battable(vaddr_t va, paddr_t *pap)
                   2113: {
                   2114: #ifdef PPC_OEA601
                   2115:        if ((MFPVR() >> 16) == MPC601)
                   2116:                return pmap_extract_battable601(va, pap);
                   2117: #endif /* PPC_OEA601 */
                   2118:
                   2119:        if (oeacpufeat & OEACPU_NOBAT)
                   2120:                return false;
                   2121:
                   2122:        const register_t batu = battable[BAT_VA2IDX(va)].batu;
                   2123:
                   2124:        if (BAT_VALID_P(batu, 0) && BAT_VA_MATCH_P(batu, va)) {
                   2125:                const register_t batl = battable[BAT_VA2IDX(va)].batl;
                   2126:                const register_t mask =
                   2127:                    (~(batu & (BAT_XBL|BAT_BL)) << 15) & ~0x1ffffL;
                   2128:                if (pap)
                   2129:                        *pap = (batl & mask) | (va & ~mask);
                   2130:                return true;
                   2131:        }
                   2132:        return false;
                   2133: }
                   2134: #endif /* PMAP_OEA */
                   2135:
1.1       matt     2136: /*
                   2137:  * Get the physical page address for the given pmap/virtual address.
                   2138:  */
1.44      thorpej  2139: bool
1.1       matt     2140: pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap)
                   2141: {
                   2142:        struct pvo_entry *pvo;
1.2       matt     2143:        register_t msr;
1.7       matt     2144:
1.50      ad       2145:        PMAP_LOCK();
1.38      sanjayl  2146:
1.7       matt     2147:        /*
1.104     thorpej  2148:         * If this is the kernel pmap, check the battable and I/O
                   2149:         * segments for a hit.  This is done only for regions outside
                   2150:         * VM_MIN_KERNEL_ADDRESS-VM_MAX_KERNEL_ADDRESS.
                   2151:         *
                   2152:         * Be careful when checking VM_MAX_KERNEL_ADDRESS; you don't
                   2153:         * want to wrap around to 0.
1.7       matt     2154:         */
                   2155:        if (pm == pmap_kernel() &&
                   2156:            (va < VM_MIN_KERNEL_ADDRESS ||
                   2157:             (KERNEL2_SR < 15 && VM_MAX_KERNEL_ADDRESS <= va))) {
1.8       matt     2158:                KASSERT((va >> ADDR_SR_SHFT) != USER_SR);
1.104     thorpej  2159: #if defined(PMAP_OEA)
1.55      garbled  2160: #ifdef PPC_OEA601
1.104     thorpej  2161:                if (pmap_extract_ioseg601(va, pap)) {
                   2162:                        PMAP_UNLOCK();
                   2163:                        return true;
                   2164:                }
1.55      garbled  2165: #endif /* PPC_OEA601 */
1.104     thorpej  2166:                if (pmap_extract_battable(va, pap)) {
                   2167:                        PMAP_UNLOCK();
                   2168:                        return true;
1.7       matt     2169:                }
1.104     thorpej  2170:                /*
                   2171:                 * We still check the HTAB...
                   2172:                 */
                   2173: #elif defined(PMAP_OEA64_BRIDGE)
                   2174:                if (va < SEGMENT_LENGTH) {
                   2175:                        if (pap)
                   2176:                                *pap = va;
1.52      garbled  2177:                        PMAP_UNLOCK();
                   2178:                        return true;
1.104     thorpej  2179:                }
                   2180:                /*
                   2181:                 * We still check the HTAB...
                   2182:                 */
                   2183: #elif defined(PMAP_OEA64)
1.38      sanjayl  2184: #error PPC_OEA64 not supported
                   2185: #endif /* PPC_OEA */
1.7       matt     2186:        }
1.1       matt     2187:
                   2188:        msr = pmap_interrupts_off();
                   2189:        pvo = pmap_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
                   2190:        if (pvo != NULL) {
                   2191:                PMAP_PVO_CHECK(pvo);            /* sanity check */
1.29      briggs   2192:                if (pap)
                   2193:                        *pap = (pvo->pvo_pte.pte_lo & PTE_RPGN)
                   2194:                            | (va & ADDR_POFF);
1.1       matt     2195:        }
                   2196:        pmap_interrupts_restore(msr);
1.50      ad       2197:        PMAP_UNLOCK();
1.1       matt     2198:        return pvo != NULL;
                   2199: }
                   2200:
                   2201: /*
                   2202:  * Lower the protection on the specified range of this pmap.
                   2203:  */
                   2204: void
                   2205: pmap_protect(pmap_t pm, vaddr_t va, vaddr_t endva, vm_prot_t prot)
                   2206: {
                   2207:        struct pvo_entry *pvo;
1.2       matt     2208:        volatile struct pte *pt;
                   2209:        register_t msr;
1.1       matt     2210:        int pteidx;
                   2211:
                   2212:        /*
                   2213:         * Since this routine only downgrades protection, we should
1.14      chs      2214:         * always be called with at least one bit not set.
1.1       matt     2215:         */
1.14      chs      2216:        KASSERT(prot != VM_PROT_ALL);
1.1       matt     2217:
                   2218:        /*
                   2219:         * If there is no protection, this is equivalent to
                   2220:         * remove the pmap from the pmap.
                   2221:         */
                   2222:        if ((prot & VM_PROT_READ) == 0) {
                   2223:                pmap_remove(pm, va, endva);
                   2224:                return;
                   2225:        }
                   2226:
1.50      ad       2227:        PMAP_LOCK();
                   2228:
1.1       matt     2229:        msr = pmap_interrupts_off();
1.6       thorpej  2230:        for (; va < endva; va += PAGE_SIZE) {
1.1       matt     2231:                pvo = pmap_pvo_find_va(pm, va, &pteidx);
                   2232:                if (pvo == NULL)
                   2233:                        continue;
                   2234:                PMAP_PVO_CHECK(pvo);            /* sanity check */
                   2235:
                   2236:                /*
                   2237:                 * Revoke executable if asked to do so.
                   2238:                 */
                   2239:                if ((prot & VM_PROT_EXECUTE) == 0)
1.14      chs      2240:                        pvo_clear_exec(pvo);
1.1       matt     2241:
                   2242: #if 0
                   2243:                /*
                   2244:                 * If the page is already read-only, no change
                   2245:                 * needs to be made.
                   2246:                 */
                   2247:                if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR)
                   2248:                        continue;
                   2249: #endif
                   2250:                /*
                   2251:                 * Grab the PTE pointer before we diddle with
                   2252:                 * the cached PTE copy.
                   2253:                 */
                   2254:                pt = pmap_pvo_to_pte(pvo, pteidx);
                   2255:                /*
                   2256:                 * Change the protection of the page.
                   2257:                 */
                   2258:                pvo->pvo_pte.pte_lo &= ~PTE_PP;
                   2259:                pvo->pvo_pte.pte_lo |= PTE_BR;
                   2260:
                   2261:                /*
                   2262:                 * If the PVO is in the page table, update
                   2263:                 * that pte at well.
                   2264:                 */
                   2265:                if (pt != NULL) {
                   2266:                        pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
1.12      matt     2267:                        PVO_WHERE(pvo, PMAP_PROTECT);
1.1       matt     2268:                        PMAPCOUNT(ptes_changed);
                   2269:                }
                   2270:
                   2271:                PMAP_PVO_CHECK(pvo);            /* sanity check */
                   2272:        }
                   2273:        pmap_interrupts_restore(msr);
1.50      ad       2274:        PMAP_UNLOCK();
1.1       matt     2275: }
                   2276:
                   2277: void
                   2278: pmap_unwire(pmap_t pm, vaddr_t va)
                   2279: {
                   2280:        struct pvo_entry *pvo;
1.2       matt     2281:        register_t msr;
1.1       matt     2282:
1.50      ad       2283:        PMAP_LOCK();
1.1       matt     2284:        msr = pmap_interrupts_off();
                   2285:        pvo = pmap_pvo_find_va(pm, va, NULL);
                   2286:        if (pvo != NULL) {
1.39      matt     2287:                if (PVO_WIRED_P(pvo)) {
1.1       matt     2288:                        pvo->pvo_vaddr &= ~PVO_WIRED;
                   2289:                        pm->pm_stats.wired_count--;
                   2290:                }
                   2291:                PMAP_PVO_CHECK(pvo);            /* sanity check */
                   2292:        }
                   2293:        pmap_interrupts_restore(msr);
1.50      ad       2294:        PMAP_UNLOCK();
1.1       matt     2295: }
                   2296:
1.108     riastrad 2297: static void
                   2298: pmap_pp_protect(struct pmap_page *pp, paddr_t pa, vm_prot_t prot)
1.1       matt     2299: {
1.33      chs      2300:        struct pvo_head *pvo_head, pvol;
1.1       matt     2301:        struct pvo_entry *pvo, *next_pvo;
1.2       matt     2302:        volatile struct pte *pt;
                   2303:        register_t msr;
1.1       matt     2304:
1.50      ad       2305:        PMAP_LOCK();
                   2306:
1.14      chs      2307:        KASSERT(prot != VM_PROT_ALL);
1.33      chs      2308:        LIST_INIT(&pvol);
1.1       matt     2309:        msr = pmap_interrupts_off();
                   2310:
                   2311:        /*
                   2312:         * When UVM reuses a page, it does a pmap_page_protect with
                   2313:         * VM_PROT_NONE.  At that point, we can clear the exec flag
                   2314:         * since we know the page will have different contents.
                   2315:         */
                   2316:        if ((prot & VM_PROT_READ) == 0) {
1.85      matt     2317:                DPRINTFN(EXEC, "[pmap_page_protect: %#" _PRIxpa ": clear-exec]\n",
1.108     riastrad 2318:                    pa);
                   2319:                if (pmap_pp_attr_fetch(pp) & PTE_EXEC) {
1.1       matt     2320:                        PMAPCOUNT(exec_uncached_page_protect);
1.108     riastrad 2321:                        pmap_pp_attr_clear(pp, PTE_EXEC);
1.1       matt     2322:                }
                   2323:        }
                   2324:
1.108     riastrad 2325:        pvo_head = &pp->pp_pvoh;
1.1       matt     2326:        for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) {
                   2327:                next_pvo = LIST_NEXT(pvo, pvo_vlink);
                   2328:                PMAP_PVO_CHECK(pvo);            /* sanity check */
                   2329:
                   2330:                /*
                   2331:                 * Downgrading to no mapping at all, we just remove the entry.
                   2332:                 */
                   2333:                if ((prot & VM_PROT_READ) == 0) {
1.33      chs      2334:                        pmap_pvo_remove(pvo, -1, &pvol);
1.1       matt     2335:                        continue;
                   2336:                }
                   2337:
                   2338:                /*
                   2339:                 * If EXEC permission is being revoked, just clear the
                   2340:                 * flag in the PVO.
                   2341:                 */
                   2342:                if ((prot & VM_PROT_EXECUTE) == 0)
1.14      chs      2343:                        pvo_clear_exec(pvo);
1.1       matt     2344:
                   2345:                /*
                   2346:                 * If this entry is already RO, don't diddle with the
                   2347:                 * page table.
                   2348:                 */
                   2349:                if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) {
                   2350:                        PMAP_PVO_CHECK(pvo);
                   2351:                        continue;
                   2352:                }
                   2353:
                   2354:                /*
                   2355:                 * Grab the PTE before the we diddle the bits so
                   2356:                 * pvo_to_pte can verify the pte contents are as
                   2357:                 * expected.
                   2358:                 */
                   2359:                pt = pmap_pvo_to_pte(pvo, -1);
                   2360:                pvo->pvo_pte.pte_lo &= ~PTE_PP;
                   2361:                pvo->pvo_pte.pte_lo |= PTE_BR;
                   2362:                if (pt != NULL) {
                   2363:                        pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
1.12      matt     2364:                        PVO_WHERE(pvo, PMAP_PAGE_PROTECT);
1.1       matt     2365:                        PMAPCOUNT(ptes_changed);
                   2366:                }
                   2367:                PMAP_PVO_CHECK(pvo);            /* sanity check */
                   2368:        }
                   2369:        pmap_interrupts_restore(msr);
1.33      chs      2370:        pmap_pvo_free_list(&pvol);
1.50      ad       2371:
                   2372:        PMAP_UNLOCK();
1.1       matt     2373: }
                   2374:
                   2375: /*
1.108     riastrad 2376:  * Lower the protection on the specified physical page.
                   2377:  */
                   2378: void
                   2379: pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
                   2380: {
                   2381:        struct vm_page_md *md = VM_PAGE_TO_MD(pg);
                   2382:
                   2383:        pmap_pp_protect(&md->mdpg_pp, VM_PAGE_TO_PHYS(pg), prot);
                   2384: }
                   2385:
                   2386: /*
                   2387:  * Lower the protection on the physical page at the specified physical
                   2388:  * address, which may not be managed and so may not have a struct
                   2389:  * vm_page.
                   2390:  */
                   2391: void
                   2392: pmap_pv_protect(paddr_t pa, vm_prot_t prot)
                   2393: {
                   2394:        struct pmap_page *pp;
                   2395:
                   2396:        if ((pp = pmap_pv_tracked(pa)) == NULL)
                   2397:                return;
                   2398:        pmap_pp_protect(pp, pa, prot);
                   2399: }
                   2400:
                   2401: /*
1.1       matt     2402:  * Activate the address space for the specified process.  If the process
                   2403:  * is the current process, load the new MMU context.
                   2404:  */
                   2405: void
                   2406: pmap_activate(struct lwp *l)
                   2407: {
1.69      rmind    2408:        struct pcb *pcb = lwp_getpcb(l);
1.1       matt     2409:        pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap;
                   2410:
                   2411:        DPRINTFN(ACTIVATE,
1.85      matt     2412:            "pmap_activate: lwp %p (curlwp %p)\n", l, curlwp);
1.1       matt     2413:
                   2414:        /*
1.70      skrll    2415:         * XXX Normally performed in cpu_lwp_fork().
1.1       matt     2416:         */
1.13      matt     2417:        pcb->pcb_pm = pmap;
1.17      matt     2418:
                   2419:        /*
                   2420:        * In theory, the SR registers need only be valid on return
                   2421:        * to user space wait to do them there.
                   2422:        */
                   2423:        if (l == curlwp) {
                   2424:                /* Store pointer to new current pmap. */
                   2425:                curpm = pmap;
                   2426:        }
1.1       matt     2427: }
                   2428:
                   2429: /*
                   2430:  * Deactivate the specified process's address space.
                   2431:  */
                   2432: void
                   2433: pmap_deactivate(struct lwp *l)
                   2434: {
                   2435: }
                   2436:
1.44      thorpej  2437: bool
1.1       matt     2438: pmap_query_bit(struct vm_page *pg, int ptebit)
                   2439: {
                   2440:        struct pvo_entry *pvo;
1.2       matt     2441:        volatile struct pte *pt;
                   2442:        register_t msr;
1.1       matt     2443:
1.50      ad       2444:        PMAP_LOCK();
                   2445:
                   2446:        if (pmap_attr_fetch(pg) & ptebit) {
                   2447:                PMAP_UNLOCK();
1.45      thorpej  2448:                return true;
1.50      ad       2449:        }
1.14      chs      2450:
1.1       matt     2451:        msr = pmap_interrupts_off();
                   2452:        LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) {
                   2453:                PMAP_PVO_CHECK(pvo);            /* sanity check */
                   2454:                /*
                   2455:                 * See if we saved the bit off.  If so cache, it and return
                   2456:                 * success.
                   2457:                 */
                   2458:                if (pvo->pvo_pte.pte_lo & ptebit) {
                   2459:                        pmap_attr_save(pg, ptebit);
                   2460:                        PMAP_PVO_CHECK(pvo);            /* sanity check */
                   2461:                        pmap_interrupts_restore(msr);
1.50      ad       2462:                        PMAP_UNLOCK();
1.45      thorpej  2463:                        return true;
1.1       matt     2464:                }
                   2465:        }
                   2466:        /*
                   2467:         * No luck, now go thru the hard part of looking at the ptes
                   2468:         * themselves.  Sync so any pending REF/CHG bits are flushed
                   2469:         * to the PTEs.
                   2470:         */
                   2471:        SYNC();
                   2472:        LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) {
                   2473:                PMAP_PVO_CHECK(pvo);            /* sanity check */
                   2474:                /*
                   2475:                 * See if this pvo have a valid PTE.  If so, fetch the
                   2476:                 * REF/CHG bits from the valid PTE.  If the appropriate
                   2477:                 * ptebit is set, cache, it and return success.
                   2478:                 */
                   2479:                pt = pmap_pvo_to_pte(pvo, -1);
                   2480:                if (pt != NULL) {
                   2481:                        pmap_pte_synch(pt, &pvo->pvo_pte);
                   2482:                        if (pvo->pvo_pte.pte_lo & ptebit) {
                   2483:                                pmap_attr_save(pg, ptebit);
                   2484:                                PMAP_PVO_CHECK(pvo);            /* sanity check */
                   2485:                                pmap_interrupts_restore(msr);
1.50      ad       2486:                                PMAP_UNLOCK();
1.45      thorpej  2487:                                return true;
1.1       matt     2488:                        }
                   2489:                }
                   2490:        }
                   2491:        pmap_interrupts_restore(msr);
1.50      ad       2492:        PMAP_UNLOCK();
1.45      thorpej  2493:        return false;
1.1       matt     2494: }
                   2495:
1.44      thorpej  2496: bool
1.1       matt     2497: pmap_clear_bit(struct vm_page *pg, int ptebit)
                   2498: {
                   2499:        struct pvo_head *pvoh = vm_page_to_pvoh(pg);
                   2500:        struct pvo_entry *pvo;
1.2       matt     2501:        volatile struct pte *pt;
                   2502:        register_t msr;
1.1       matt     2503:        int rv = 0;
                   2504:
1.50      ad       2505:        PMAP_LOCK();
1.1       matt     2506:        msr = pmap_interrupts_off();
                   2507:
                   2508:        /*
                   2509:         * Fetch the cache value
                   2510:         */
                   2511:        rv |= pmap_attr_fetch(pg);
                   2512:
                   2513:        /*
                   2514:         * Clear the cached value.
                   2515:         */
                   2516:        pmap_attr_clear(pg, ptebit);
                   2517:
                   2518:        /*
                   2519:         * Sync so any pending REF/CHG bits are flushed to the PTEs (so we
                   2520:         * can reset the right ones).  Note that since the pvo entries and
                   2521:         * list heads are accessed via BAT0 and are never placed in the
                   2522:         * page table, we don't have to worry about further accesses setting
                   2523:         * the REF/CHG bits.
                   2524:         */
                   2525:        SYNC();
                   2526:
                   2527:        /*
                   2528:         * For each pvo entry, clear pvo's ptebit.  If this pvo have a
                   2529:         * valid PTE.  If so, clear the ptebit from the valid PTE.
                   2530:         */
                   2531:        LIST_FOREACH(pvo, pvoh, pvo_vlink) {
                   2532:                PMAP_PVO_CHECK(pvo);            /* sanity check */
                   2533:                pt = pmap_pvo_to_pte(pvo, -1);
                   2534:                if (pt != NULL) {
                   2535:                        /*
                   2536:                         * Only sync the PTE if the bit we are looking
                   2537:                         * for is not already set.
                   2538:                         */
                   2539:                        if ((pvo->pvo_pte.pte_lo & ptebit) == 0)
                   2540:                                pmap_pte_synch(pt, &pvo->pvo_pte);
                   2541:                        /*
                   2542:                         * If the bit we are looking for was already set,
                   2543:                         * clear that bit in the pte.
                   2544:                         */
                   2545:                        if (pvo->pvo_pte.pte_lo & ptebit)
                   2546:                                pmap_pte_clear(pt, PVO_VADDR(pvo), ptebit);
                   2547:                }
                   2548:                rv |= pvo->pvo_pte.pte_lo & (PTE_CHG|PTE_REF);
                   2549:                pvo->pvo_pte.pte_lo &= ~ptebit;
                   2550:                PMAP_PVO_CHECK(pvo);            /* sanity check */
                   2551:        }
                   2552:        pmap_interrupts_restore(msr);
1.14      chs      2553:
1.1       matt     2554:        /*
                   2555:         * If we are clearing the modify bit and this page was marked EXEC
                   2556:         * and the user of the page thinks the page was modified, then we
                   2557:         * need to clean it from the icache if it's mapped or clear the EXEC
                   2558:         * bit if it's not mapped.  The page itself might not have the CHG
                   2559:         * bit set if the modification was done via DMA to the page.
                   2560:         */
                   2561:        if ((ptebit & PTE_CHG) && (rv & PTE_EXEC)) {
                   2562:                if (LIST_EMPTY(pvoh)) {
1.85      matt     2563:                        DPRINTFN(EXEC, "[pmap_clear_bit: %#" _PRIxpa ": clear-exec]\n",
                   2564:                            VM_PAGE_TO_PHYS(pg));
1.1       matt     2565:                        pmap_attr_clear(pg, PTE_EXEC);
                   2566:                        PMAPCOUNT(exec_uncached_clear_modify);
                   2567:                } else {
1.85      matt     2568:                        DPRINTFN(EXEC, "[pmap_clear_bit: %#" _PRIxpa ": syncicache]\n",
                   2569:                            VM_PAGE_TO_PHYS(pg));
1.34      yamt     2570:                        pmap_syncicache(VM_PAGE_TO_PHYS(pg), PAGE_SIZE);
1.1       matt     2571:                        PMAPCOUNT(exec_synced_clear_modify);
                   2572:                }
                   2573:        }
1.50      ad       2574:        PMAP_UNLOCK();
1.1       matt     2575:        return (rv & ptebit) != 0;
                   2576: }
                   2577:
                   2578: void
                   2579: pmap_procwr(struct proc *p, vaddr_t va, size_t len)
                   2580: {
                   2581:        struct pvo_entry *pvo;
                   2582:        size_t offset = va & ADDR_POFF;
                   2583:        int s;
                   2584:
1.50      ad       2585:        PMAP_LOCK();
1.1       matt     2586:        s = splvm();
                   2587:        while (len > 0) {
1.6       thorpej  2588:                size_t seglen = PAGE_SIZE - offset;
1.1       matt     2589:                if (seglen > len)
                   2590:                        seglen = len;
                   2591:                pvo = pmap_pvo_find_va(p->p_vmspace->vm_map.pmap, va, NULL);
1.39      matt     2592:                if (pvo != NULL && PVO_EXECUTABLE_P(pvo)) {
1.1       matt     2593:                        pmap_syncicache(
                   2594:                            (pvo->pvo_pte.pte_lo & PTE_RPGN) | offset, seglen);
                   2595:                        PMAP_PVO_CHECK(pvo);
                   2596:                }
                   2597:                va += seglen;
                   2598:                len -= seglen;
                   2599:                offset = 0;
                   2600:        }
                   2601:        splx(s);
1.50      ad       2602:        PMAP_UNLOCK();
1.1       matt     2603: }
                   2604:
                   2605: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
                   2606: void
1.2       matt     2607: pmap_pte_print(volatile struct pte *pt)
1.1       matt     2608: {
                   2609:        printf("PTE %p: ", pt);
1.38      sanjayl  2610:
1.53      garbled  2611: #if defined(PMAP_OEA)
1.1       matt     2612:        /* High word: */
1.54      mlelstv  2613:        printf("%#" _PRIxpte ": [", pt->pte_hi);
1.53      garbled  2614: #else
1.54      mlelstv  2615:        printf("%#" _PRIxpte ": [", pt->pte_hi);
1.53      garbled  2616: #endif /* PMAP_OEA */
1.38      sanjayl  2617:
1.1       matt     2618:        printf("%c ", (pt->pte_hi & PTE_VALID) ? 'v' : 'i');
                   2619:        printf("%c ", (pt->pte_hi & PTE_HID) ? 'h' : '-');
1.38      sanjayl  2620:
1.54      mlelstv  2621:        printf("%#" _PRIxpte " %#" _PRIxpte "",
1.38      sanjayl  2622:            (pt->pte_hi &~ PTE_VALID)>>PTE_VSID_SHFT,
                   2623:            pt->pte_hi & PTE_API);
1.53      garbled  2624: #if defined(PMAP_OEA) || defined(PMAP_OEA64_BRIDGE)
1.54      mlelstv  2625:        printf(" (va %#" _PRIxva ")] ", pmap_pte_to_va(pt));
1.38      sanjayl  2626: #else
1.54      mlelstv  2627:        printf(" (va %#" _PRIxva ")] ", pmap_pte_to_va(pt));
1.53      garbled  2628: #endif /* PMAP_OEA */
1.38      sanjayl  2629:
1.1       matt     2630:        /* Low word: */
1.53      garbled  2631: #if defined (PMAP_OEA)
1.54      mlelstv  2632:        printf(" %#" _PRIxpte ": [", pt->pte_lo);
                   2633:        printf("%#" _PRIxpte "... ", pt->pte_lo >> 12);
1.53      garbled  2634: #else
1.54      mlelstv  2635:        printf(" %#" _PRIxpte ": [", pt->pte_lo);
                   2636:        printf("%#" _PRIxpte "... ", pt->pte_lo >> 12);
1.38      sanjayl  2637: #endif
1.1       matt     2638:        printf("%c ", (pt->pte_lo & PTE_REF) ? 'r' : 'u');
                   2639:        printf("%c ", (pt->pte_lo & PTE_CHG) ? 'c' : 'n');
                   2640:        printf("%c", (pt->pte_lo & PTE_W) ? 'w' : '.');
                   2641:        printf("%c", (pt->pte_lo & PTE_I) ? 'i' : '.');
                   2642:        printf("%c", (pt->pte_lo & PTE_M) ? 'm' : '.');
                   2643:        printf("%c ", (pt->pte_lo & PTE_G) ? 'g' : '.');
                   2644:        switch (pt->pte_lo & PTE_PP) {
                   2645:        case PTE_BR: printf("br]\n"); break;
                   2646:        case PTE_BW: printf("bw]\n"); break;
                   2647:        case PTE_SO: printf("so]\n"); break;
                   2648:        case PTE_SW: printf("sw]\n"); break;
                   2649:        }
                   2650: }
                   2651: #endif
                   2652:
                   2653: #if defined(DDB)
                   2654: void
                   2655: pmap_pteg_check(void)
                   2656: {
1.2       matt     2657:        volatile struct pte *pt;
1.1       matt     2658:        int i;
                   2659:        int ptegidx;
                   2660:        u_int p_valid = 0;
                   2661:        u_int s_valid = 0;
                   2662:        u_int invalid = 0;
1.38      sanjayl  2663:
1.1       matt     2664:        for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) {
                   2665:                for (pt = pmap_pteg_table[ptegidx].pt, i = 8; --i >= 0; pt++) {
                   2666:                        if (pt->pte_hi & PTE_VALID) {
                   2667:                                if (pt->pte_hi & PTE_HID)
                   2668:                                        s_valid++;
                   2669:                                else
1.38      sanjayl  2670:                                {
1.1       matt     2671:                                        p_valid++;
1.38      sanjayl  2672:                                }
1.1       matt     2673:                        } else
                   2674:                                invalid++;
                   2675:                }
                   2676:        }
                   2677:        printf("pteg_check: v(p) %#x (%d), v(s) %#x (%d), i %#x (%d)\n",
                   2678:                p_valid, p_valid, s_valid, s_valid,
                   2679:                invalid, invalid);
                   2680: }
                   2681:
                   2682: void
                   2683: pmap_print_mmuregs(void)
                   2684: {
                   2685:        int i;
1.97      rin      2686: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1       matt     2687:        u_int cpuvers;
1.90      mrg      2688: #endif
1.53      garbled  2689: #ifndef PMAP_OEA64
1.1       matt     2690:        vaddr_t addr;
1.2       matt     2691:        register_t soft_sr[16];
1.18      matt     2692: #endif
1.97      rin      2693: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1       matt     2694:        struct bat soft_ibat[4];
                   2695:        struct bat soft_dbat[4];
1.38      sanjayl  2696: #endif
1.53      garbled  2697:        paddr_t sdr1;
1.1       matt     2698:
1.97      rin      2699: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1       matt     2700:        cpuvers = MFPVR() >> 16;
1.90      mrg      2701: #endif
1.35      perry    2702:        __asm volatile ("mfsdr1 %0" : "=r"(sdr1));
1.53      garbled  2703: #ifndef PMAP_OEA64
1.16      kleink   2704:        addr = 0;
1.27      chs      2705:        for (i = 0; i < 16; i++) {
1.1       matt     2706:                soft_sr[i] = MFSRIN(addr);
                   2707:                addr += (1 << ADDR_SR_SHFT);
                   2708:        }
1.18      matt     2709: #endif
1.1       matt     2710:
1.97      rin      2711: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1       matt     2712:        /* read iBAT (601: uBAT) registers */
1.35      perry    2713:        __asm volatile ("mfibatu %0,0" : "=r"(soft_ibat[0].batu));
                   2714:        __asm volatile ("mfibatl %0,0" : "=r"(soft_ibat[0].batl));
                   2715:        __asm volatile ("mfibatu %0,1" : "=r"(soft_ibat[1].batu));
                   2716:        __asm volatile ("mfibatl %0,1" : "=r"(soft_ibat[1].batl));
                   2717:        __asm volatile ("mfibatu %0,2" : "=r"(soft_ibat[2].batu));
                   2718:        __asm volatile ("mfibatl %0,2" : "=r"(soft_ibat[2].batl));
                   2719:        __asm volatile ("mfibatu %0,3" : "=r"(soft_ibat[3].batu));
                   2720:        __asm volatile ("mfibatl %0,3" : "=r"(soft_ibat[3].batl));
1.1       matt     2721:
                   2722:
                   2723:        if (cpuvers != MPC601) {
                   2724:                /* read dBAT registers */
1.35      perry    2725:                __asm volatile ("mfdbatu %0,0" : "=r"(soft_dbat[0].batu));
                   2726:                __asm volatile ("mfdbatl %0,0" : "=r"(soft_dbat[0].batl));
                   2727:                __asm volatile ("mfdbatu %0,1" : "=r"(soft_dbat[1].batu));
                   2728:                __asm volatile ("mfdbatl %0,1" : "=r"(soft_dbat[1].batl));
                   2729:                __asm volatile ("mfdbatu %0,2" : "=r"(soft_dbat[2].batu));
                   2730:                __asm volatile ("mfdbatl %0,2" : "=r"(soft_dbat[2].batl));
                   2731:                __asm volatile ("mfdbatu %0,3" : "=r"(soft_dbat[3].batu));
                   2732:                __asm volatile ("mfdbatl %0,3" : "=r"(soft_dbat[3].batl));
1.1       matt     2733:        }
1.38      sanjayl  2734: #endif
1.1       matt     2735:
1.54      mlelstv  2736:        printf("SDR1:\t%#" _PRIxpa "\n", sdr1);
1.53      garbled  2737: #ifndef PMAP_OEA64
1.1       matt     2738:        printf("SR[]:\t");
1.27      chs      2739:        for (i = 0; i < 4; i++)
1.53      garbled  2740:                printf("0x%08lx,   ", soft_sr[i]);
1.1       matt     2741:        printf("\n\t");
1.27      chs      2742:        for ( ; i < 8; i++)
1.53      garbled  2743:                printf("0x%08lx,   ", soft_sr[i]);
1.1       matt     2744:        printf("\n\t");
1.27      chs      2745:        for ( ; i < 12; i++)
1.53      garbled  2746:                printf("0x%08lx,   ", soft_sr[i]);
1.1       matt     2747:        printf("\n\t");
1.27      chs      2748:        for ( ; i < 16; i++)
1.53      garbled  2749:                printf("0x%08lx,   ", soft_sr[i]);
1.1       matt     2750:        printf("\n");
1.18      matt     2751: #endif
1.1       matt     2752:
1.97      rin      2753: #if defined(PMAP_OEA) || defined(PMAP_OEA64_BRIDGE)
1.1       matt     2754:        printf("%cBAT[]:\t", cpuvers == MPC601 ? 'u' : 'i');
1.27      chs      2755:        for (i = 0; i < 4; i++) {
1.2       matt     2756:                printf("0x%08lx 0x%08lx, ",
1.1       matt     2757:                        soft_ibat[i].batu, soft_ibat[i].batl);
                   2758:                if (i == 1)
                   2759:                        printf("\n\t");
                   2760:        }
                   2761:        if (cpuvers != MPC601) {
                   2762:                printf("\ndBAT[]:\t");
1.27      chs      2763:                for (i = 0; i < 4; i++) {
1.2       matt     2764:                        printf("0x%08lx 0x%08lx, ",
1.1       matt     2765:                                soft_dbat[i].batu, soft_dbat[i].batl);
                   2766:                        if (i == 1)
                   2767:                                printf("\n\t");
                   2768:                }
                   2769:        }
                   2770:        printf("\n");
1.53      garbled  2771: #endif /* PMAP_OEA... */
1.1       matt     2772: }
                   2773:
                   2774: void
                   2775: pmap_print_pte(pmap_t pm, vaddr_t va)
                   2776: {
                   2777:        struct pvo_entry *pvo;
1.2       matt     2778:        volatile struct pte *pt;
1.1       matt     2779:        int pteidx;
                   2780:
                   2781:        pvo = pmap_pvo_find_va(pm, va, &pteidx);
                   2782:        if (pvo != NULL) {
                   2783:                pt = pmap_pvo_to_pte(pvo, pteidx);
                   2784:                if (pt != NULL) {
1.53      garbled  2785:                        printf("VA %#" _PRIxva " -> %p -> %s %#" _PRIxpte ", %#" _PRIxpte "\n",
1.38      sanjayl  2786:                                va, pt,
                   2787:                                pt->pte_hi & PTE_HID ? "(sec)" : "(pri)",
                   2788:                                pt->pte_hi, pt->pte_lo);
1.1       matt     2789:                } else {
                   2790:                        printf("No valid PTE found\n");
                   2791:                }
                   2792:        } else {
                   2793:                printf("Address not in pmap\n");
                   2794:        }
                   2795: }
                   2796:
                   2797: void
                   2798: pmap_pteg_dist(void)
                   2799: {
                   2800:        struct pvo_entry *pvo;
                   2801:        int ptegidx;
                   2802:        int depth;
                   2803:        int max_depth = 0;
                   2804:        unsigned int depths[64];
                   2805:
                   2806:        memset(depths, 0, sizeof(depths));
                   2807:        for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) {
                   2808:                depth = 0;
                   2809:                TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
                   2810:                        depth++;
                   2811:                }
                   2812:                if (depth > max_depth)
                   2813:                        max_depth = depth;
                   2814:                if (depth > 63)
                   2815:                        depth = 63;
                   2816:                depths[depth]++;
                   2817:        }
                   2818:
                   2819:        for (depth = 0; depth < 64; depth++) {
                   2820:                printf("  [%2d]: %8u", depth, depths[depth]);
                   2821:                if ((depth & 3) == 3)
                   2822:                        printf("\n");
                   2823:                if (depth == max_depth)
                   2824:                        break;
                   2825:        }
                   2826:        if ((depth & 3) != 3)
                   2827:                printf("\n");
                   2828:        printf("Max depth found was %d\n", max_depth);
                   2829: }
                   2830: #endif /* DEBUG */
                   2831:
                   2832: #if defined(PMAPCHECK) || defined(DEBUG)
                   2833: void
                   2834: pmap_pvo_verify(void)
                   2835: {
                   2836:        int ptegidx;
                   2837:        int s;
                   2838:
                   2839:        s = splvm();
                   2840:        for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) {
                   2841:                struct pvo_entry *pvo;
                   2842:                TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
                   2843:                        if ((uintptr_t) pvo >= SEGMENT_LENGTH)
                   2844:                                panic("pmap_pvo_verify: invalid pvo %p "
                   2845:                                    "on list %#x", pvo, ptegidx);
                   2846:                        pmap_pvo_check(pvo);
                   2847:                }
                   2848:        }
                   2849:        splx(s);
                   2850: }
                   2851: #endif /* PMAPCHECK */
                   2852:
                   2853: void *
1.106     martin   2854: pmap_pool_alloc(struct pool *pp, int flags)
1.1       matt     2855: {
                   2856:        struct pvo_page *pvop;
1.106     martin   2857:        struct vm_page *pg;
1.1       matt     2858:
1.50      ad       2859:        if (uvm.page_init_done != true) {
                   2860:                return (void *) uvm_pageboot_alloc(PAGE_SIZE);
                   2861:        }
                   2862:
                   2863:        PMAP_LOCK();
1.106     martin   2864:        pvop = SIMPLEQ_FIRST(&pmap_pvop_head);
1.1       matt     2865:        if (pvop != NULL) {
1.106     martin   2866:                pmap_pvop_free--;
                   2867:                SIMPLEQ_REMOVE_HEAD(&pmap_pvop_head, pvop_link);
1.50      ad       2868:                PMAP_UNLOCK();
1.1       matt     2869:                return pvop;
                   2870:        }
1.50      ad       2871:        PMAP_UNLOCK();
1.1       matt     2872:  again:
                   2873:        pg = uvm_pagealloc_strat(NULL, 0, NULL, UVM_PGA_USERESERVE,
                   2874:            UVM_PGA_STRAT_ONLY, VM_FREELIST_FIRST256);
                   2875:        if (__predict_false(pg == NULL)) {
                   2876:                if (flags & PR_WAITOK) {
                   2877:                        uvm_wait("plpg");
                   2878:                        goto again;
                   2879:                } else {
                   2880:                        return (0);
                   2881:                }
                   2882:        }
1.53      garbled  2883:        KDASSERT(VM_PAGE_TO_PHYS(pg) == (uintptr_t)VM_PAGE_TO_PHYS(pg));
                   2884:        return (void *)(uintptr_t) VM_PAGE_TO_PHYS(pg);
1.1       matt     2885: }
                   2886:
                   2887: void
1.106     martin   2888: pmap_pool_free(struct pool *pp, void *va)
1.1       matt     2889: {
                   2890:        struct pvo_page *pvop;
                   2891:
1.50      ad       2892:        PMAP_LOCK();
1.1       matt     2893:        pvop = va;
1.106     martin   2894:        SIMPLEQ_INSERT_HEAD(&pmap_pvop_head, pvop, pvop_link);
                   2895:        pmap_pvop_free++;
                   2896:        if (pmap_pvop_free > pmap_pvop_maxfree)
                   2897:                pmap_pvop_maxfree = pmap_pvop_free;
1.50      ad       2898:        PMAP_UNLOCK();
1.1       matt     2899: #if 0
                   2900:        uvm_pagefree(PHYS_TO_VM_PAGE((paddr_t) va));
                   2901: #endif
                   2902: }
                   2903:
                   2904: /*
                   2905:  * This routine in bootstraping to steal to-be-managed memory (which will
                   2906:  * then be unmanaged).  We use it to grab from the first 256MB for our
                   2907:  * pmap needs and above 256MB for other stuff.
                   2908:  */
                   2909: vaddr_t
1.10      thorpej  2910: pmap_steal_memory(vsize_t vsize, vaddr_t *vstartp, vaddr_t *vendp)
1.1       matt     2911: {
                   2912:        vsize_t size;
                   2913:        vaddr_t va;
1.94      cherry   2914:        paddr_t start, end, pa = 0;
                   2915:        int npgs, freelist;
                   2916:        uvm_physseg_t bank;
1.1       matt     2917:
1.45      thorpej  2918:        if (uvm.page_init_done == true)
1.1       matt     2919:                panic("pmap_steal_memory: called _after_ bootstrap");
                   2920:
1.10      thorpej  2921:        *vstartp = VM_MIN_KERNEL_ADDRESS;
                   2922:        *vendp = VM_MAX_KERNEL_ADDRESS;
                   2923:
1.1       matt     2924:        size = round_page(vsize);
                   2925:        npgs = atop(size);
                   2926:
                   2927:        /*
                   2928:         * PA 0 will never be among those given to UVM so we can use it
                   2929:         * to indicate we couldn't steal any memory.
                   2930:         */
1.94      cherry   2931:
                   2932:        for (bank = uvm_physseg_get_first();
                   2933:             uvm_physseg_valid_p(bank);
                   2934:             bank = uvm_physseg_get_next(bank)) {
                   2935:
                   2936:                freelist = uvm_physseg_get_free_list(bank);
                   2937:                start = uvm_physseg_get_start(bank);
                   2938:                end = uvm_physseg_get_end(bank);
                   2939:
                   2940:                if (freelist == VM_FREELIST_FIRST256 &&
                   2941:                    (end - start) >= npgs) {
                   2942:                        pa = ptoa(start);
1.1       matt     2943:                        break;
                   2944:                }
                   2945:        }
                   2946:
                   2947:        if (pa == 0)
                   2948:                panic("pmap_steal_memory: no approriate memory to steal!");
                   2949:
1.94      cherry   2950:        uvm_physseg_unplug(start, npgs);
1.1       matt     2951:
                   2952:        va = (vaddr_t) pa;
1.46      christos 2953:        memset((void *) va, 0, size);
1.1       matt     2954:        pmap_pages_stolen += npgs;
                   2955: #ifdef DEBUG
                   2956:        if (pmapdebug && npgs > 1) {
                   2957:                u_int cnt = 0;
1.94      cherry   2958:        for (bank = uvm_physseg_get_first();
                   2959:             uvm_physseg_valid_p(bank);
                   2960:             bank = uvm_physseg_get_next(bank)) {
                   2961:                cnt += uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank);
1.73      uebayasi 2962:                }
1.1       matt     2963:                printf("pmap_steal_memory: stole %u (total %u) pages (%u left)\n",
                   2964:                    npgs, pmap_pages_stolen, cnt);
                   2965:        }
                   2966: #endif
                   2967:
                   2968:        return va;
                   2969: }
                   2970:
                   2971: /*
                   2972:  * Find a chuck of memory with right size and alignment.
                   2973:  */
1.53      garbled  2974: paddr_t
1.1       matt     2975: pmap_boot_find_memory(psize_t size, psize_t alignment, int at_end)
                   2976: {
                   2977:        struct mem_region *mp;
                   2978:        paddr_t s, e;
                   2979:        int i, j;
                   2980:
                   2981:        size = round_page(size);
                   2982:
                   2983:        DPRINTFN(BOOT,
1.85      matt     2984:            "pmap_boot_find_memory: size=%#" _PRIxpa ", alignment=%#" _PRIxpa ", at_end=%d",
                   2985:            size, alignment, at_end);
1.1       matt     2986:
1.6       thorpej  2987:        if (alignment < PAGE_SIZE || (alignment & (alignment-1)) != 0)
1.54      mlelstv  2988:                panic("pmap_boot_find_memory: invalid alignment %#" _PRIxpa,
1.1       matt     2989:                    alignment);
                   2990:
                   2991:        if (at_end) {
1.6       thorpej  2992:                if (alignment != PAGE_SIZE)
1.1       matt     2993:                        panic("pmap_boot_find_memory: invalid ending "
1.53      garbled  2994:                            "alignment %#" _PRIxpa, alignment);
1.1       matt     2995:
                   2996:                for (mp = &avail[avail_cnt-1]; mp >= avail; mp--) {
                   2997:                        s = mp->start + mp->size - size;
                   2998:                        if (s >= mp->start && mp->size >= size) {
1.85      matt     2999:                                DPRINTFN(BOOT, ": %#" _PRIxpa "\n", s);
1.1       matt     3000:                                DPRINTFN(BOOT,
1.85      matt     3001:                                    "pmap_boot_find_memory: b-avail[%d] start "
                   3002:                                    "%#" _PRIxpa " size %#" _PRIxpa "\n", mp - avail,
                   3003:                                     mp->start, mp->size);
1.1       matt     3004:                                mp->size -= size;
                   3005:                                DPRINTFN(BOOT,
1.85      matt     3006:                                    "pmap_boot_find_memory: a-avail[%d] start "
                   3007:                                    "%#" _PRIxpa " size %#" _PRIxpa "\n", mp - avail,
                   3008:                                     mp->start, mp->size);
1.53      garbled  3009:                                return s;
1.1       matt     3010:                        }
                   3011:                }
                   3012:                panic("pmap_boot_find_memory: no available memory");
                   3013:        }
                   3014:
                   3015:        for (mp = avail, i = 0; i < avail_cnt; i++, mp++) {
                   3016:                s = (mp->start + alignment - 1) & ~(alignment-1);
                   3017:                e = s + size;
                   3018:
                   3019:                /*
                   3020:                 * Is the calculated region entirely within the region?
                   3021:                 */
                   3022:                if (s < mp->start || e > mp->start + mp->size)
                   3023:                        continue;
                   3024:
1.85      matt     3025:                DPRINTFN(BOOT, ": %#" _PRIxpa "\n", s);
1.1       matt     3026:                if (s == mp->start) {
                   3027:                        /*
                   3028:                         * If the block starts at the beginning of region,
                   3029:                         * adjust the size & start. (the region may now be
                   3030:                         * zero in length)
                   3031:                         */
                   3032:                        DPRINTFN(BOOT,
1.85      matt     3033:                            "pmap_boot_find_memory: b-avail[%d] start "
                   3034:                            "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1       matt     3035:                        mp->start += size;
                   3036:                        mp->size -= size;
                   3037:                        DPRINTFN(BOOT,
1.85      matt     3038:                            "pmap_boot_find_memory: a-avail[%d] start "
                   3039:                            "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1       matt     3040:                } else if (e == mp->start + mp->size) {
                   3041:                        /*
                   3042:                         * If the block starts at the beginning of region,
                   3043:                         * adjust only the size.
                   3044:                         */
                   3045:                        DPRINTFN(BOOT,
1.85      matt     3046:                            "pmap_boot_find_memory: b-avail[%d] start "
                   3047:                            "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1       matt     3048:                        mp->size -= size;
                   3049:                        DPRINTFN(BOOT,
1.85      matt     3050:                            "pmap_boot_find_memory: a-avail[%d] start "
                   3051:                            "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1       matt     3052:                } else {
                   3053:                        /*
                   3054:                         * Block is in the middle of the region, so we
                   3055:                         * have to split it in two.
                   3056:                         */
                   3057:                        for (j = avail_cnt; j > i + 1; j--) {
                   3058:                                avail[j] = avail[j-1];
                   3059:                        }
                   3060:                        DPRINTFN(BOOT,
1.85      matt     3061:                            "pmap_boot_find_memory: b-avail[%d] start "
                   3062:                            "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1       matt     3063:                        mp[1].start = e;
                   3064:                        mp[1].size = mp[0].start + mp[0].size - e;
                   3065:                        mp[0].size = s - mp[0].start;
                   3066:                        avail_cnt++;
                   3067:                        for (; i < avail_cnt; i++) {
                   3068:                                DPRINTFN(BOOT,
1.85      matt     3069:                                    "pmap_boot_find_memory: a-avail[%d] "
                   3070:                                    "start %#" _PRIxpa " size %#" _PRIxpa "\n", i,
                   3071:                                     avail[i].start, avail[i].size);
1.1       matt     3072:                        }
                   3073:                }
1.53      garbled  3074:                KASSERT(s == (uintptr_t) s);
                   3075:                return s;
1.1       matt     3076:        }
                   3077:        panic("pmap_boot_find_memory: not enough memory for "
1.54      mlelstv  3078:            "%#" _PRIxpa "/%#" _PRIxpa " allocation?", size, alignment);
1.1       matt     3079: }
                   3080:
1.38      sanjayl  3081: /* XXXSL: we dont have any BATs to do this, map in Segment 0 1:1 using page tables */
1.53      garbled  3082: #if defined (PMAP_OEA64_BRIDGE)
1.38      sanjayl  3083: int
                   3084: pmap_setup_segment0_map(int use_large_pages, ...)
                   3085: {
1.88      christos 3086:     vaddr_t va, va_end;
1.38      sanjayl  3087:
                   3088:     register_t pte_lo = 0x0;
1.90      mrg      3089:     int ptegidx = 0;
1.38      sanjayl  3090:     struct pte pte;
                   3091:     va_list ap;
                   3092:
                   3093:     /* Coherent + Supervisor RW, no user access */
                   3094:     pte_lo = PTE_M;
                   3095:
                   3096:     /* XXXSL
                   3097:      * Map in 1st segment 1:1, we'll be careful not to spill kernel entries later,
                   3098:      * these have to take priority.
                   3099:      */
                   3100:     for (va = 0x0; va < SEGMENT_LENGTH; va += 0x1000) {
                   3101:         ptegidx = va_to_pteg(pmap_kernel(), va);
                   3102:         pmap_pte_create(&pte, pmap_kernel(), va, va | pte_lo);
1.90      mrg      3103:         (void)pmap_pte_insert(ptegidx, &pte);
1.38      sanjayl  3104:     }
                   3105:
                   3106:     va_start(ap, use_large_pages);
                   3107:     while (1) {
                   3108:         paddr_t pa;
                   3109:         size_t size;
                   3110:
                   3111:         va = va_arg(ap, vaddr_t);
                   3112:
                   3113:         if (va == 0)
                   3114:             break;
                   3115:
                   3116:         pa = va_arg(ap, paddr_t);
                   3117:         size = va_arg(ap, size_t);
                   3118:
1.88      christos 3119:         for (va_end = va + size; va < va_end; va += 0x1000, pa += 0x1000) {
1.38      sanjayl  3120: #if 0
1.54      mlelstv  3121:            printf("%s: Inserting: va: %#" _PRIxva ", pa: %#" _PRIxpa "\n", __func__,  va, pa);
1.38      sanjayl  3122: #endif
                   3123:             ptegidx = va_to_pteg(pmap_kernel(), va);
                   3124:             pmap_pte_create(&pte, pmap_kernel(), va, pa | pte_lo);
1.90      mrg      3125:             (void)pmap_pte_insert(ptegidx, &pte);
1.38      sanjayl  3126:         }
                   3127:     }
1.93      dholland 3128:     va_end(ap);
1.38      sanjayl  3129:
                   3130:     TLBSYNC();
                   3131:     SYNC();
                   3132:     return (0);
                   3133: }
1.53      garbled  3134: #endif /* PMAP_OEA64_BRIDGE */
1.38      sanjayl  3135:
1.1       matt     3136: /*
1.99      thorpej  3137:  * Set up the bottom level of the data structures necessary for the kernel
                   3138:  * to manage memory.  MMU hardware is programmed in pmap_bootstrap2().
1.1       matt     3139:  */
                   3140: void
1.99      thorpej  3141: pmap_bootstrap1(paddr_t kernelstart, paddr_t kernelend)
1.1       matt     3142: {
                   3143:        struct mem_region *mp, tmp;
                   3144:        paddr_t s, e;
                   3145:        psize_t size;
                   3146:        int i, j;
                   3147:
                   3148:        /*
                   3149:         * Get memory.
                   3150:         */
                   3151:        mem_regions(&mem, &avail);
                   3152: #if defined(DEBUG)
                   3153:        if (pmapdebug & PMAPDEBUG_BOOT) {
                   3154:                printf("pmap_bootstrap: memory configuration:\n");
                   3155:                for (mp = mem; mp->size; mp++) {
1.54      mlelstv  3156:                        printf("pmap_bootstrap: mem start %#" _PRIxpa " size %#" _PRIxpa "\n",
1.1       matt     3157:                                mp->start, mp->size);
                   3158:                }
                   3159:                for (mp = avail; mp->size; mp++) {
1.54      mlelstv  3160:                        printf("pmap_bootstrap: avail start %#" _PRIxpa " size %#" _PRIxpa "\n",
1.1       matt     3161:                                mp->start, mp->size);
                   3162:                }
                   3163:        }
                   3164: #endif
                   3165:
                   3166:        /*
                   3167:         * Find out how much physical memory we have and in how many chunks.
                   3168:         */
                   3169:        for (mem_cnt = 0, mp = mem; mp->size; mp++) {
                   3170:                if (mp->start >= pmap_memlimit)
                   3171:                        continue;
                   3172:                if (mp->start + mp->size > pmap_memlimit) {
                   3173:                        size = pmap_memlimit - mp->start;
                   3174:                        physmem += btoc(size);
                   3175:                } else {
                   3176:                        physmem += btoc(mp->size);
                   3177:                }
                   3178:                mem_cnt++;
                   3179:        }
                   3180:
                   3181:        /*
                   3182:         * Count the number of available entries.
                   3183:         */
                   3184:        for (avail_cnt = 0, mp = avail; mp->size; mp++)
                   3185:                avail_cnt++;
                   3186:
                   3187:        /*
                   3188:         * Page align all regions.
                   3189:         */
                   3190:        kernelstart = trunc_page(kernelstart);
                   3191:        kernelend = round_page(kernelend);
                   3192:        for (mp = avail, i = 0; i < avail_cnt; i++, mp++) {
                   3193:                s = round_page(mp->start);
                   3194:                mp->size -= (s - mp->start);
                   3195:                mp->size = trunc_page(mp->size);
                   3196:                mp->start = s;
                   3197:                e = mp->start + mp->size;
                   3198:
                   3199:                DPRINTFN(BOOT,
1.85      matt     3200:                    "pmap_bootstrap: b-avail[%d] start %#" _PRIxpa " size %#" _PRIxpa "\n",
                   3201:                    i, mp->start, mp->size);
1.1       matt     3202:
                   3203:                /*
                   3204:                 * Don't allow the end to run beyond our artificial limit
                   3205:                 */
                   3206:                if (e > pmap_memlimit)
                   3207:                        e = pmap_memlimit;
                   3208:
                   3209:                /*
                   3210:                 * Is this region empty or strange?  skip it.
                   3211:                 */
                   3212:                if (e <= s) {
                   3213:                        mp->start = 0;
                   3214:                        mp->size = 0;
                   3215:                        continue;
                   3216:                }
                   3217:
                   3218:                /*
                   3219:                 * Does this overlap the beginning of kernel?
                   3220:                 *   Does extend past the end of the kernel?
                   3221:                 */
                   3222:                else if (s < kernelstart && e > kernelstart) {
                   3223:                        if (e > kernelend) {
                   3224:                                avail[avail_cnt].start = kernelend;
                   3225:                                avail[avail_cnt].size = e - kernelend;
                   3226:                                avail_cnt++;
                   3227:                        }
                   3228:                        mp->size = kernelstart - s;
                   3229:                }
                   3230:                /*
                   3231:                 * Check whether this region overlaps the end of the kernel.
                   3232:                 */
                   3233:                else if (s < kernelend && e > kernelend) {
                   3234:                        mp->start = kernelend;
                   3235:                        mp->size = e - kernelend;
                   3236:                }
                   3237:                /*
                   3238:                 * Look whether this regions is completely inside the kernel.
                   3239:                 * Nuke it if it does.
                   3240:                 */
                   3241:                else if (s >= kernelstart && e <= kernelend) {
                   3242:                        mp->start = 0;
                   3243:                        mp->size = 0;
                   3244:                }
                   3245:                /*
                   3246:                 * If the user imposed a memory limit, enforce it.
                   3247:                 */
                   3248:                else if (s >= pmap_memlimit) {
1.6       thorpej  3249:                        mp->start = -PAGE_SIZE; /* let's know why */
1.1       matt     3250:                        mp->size = 0;
                   3251:                }
                   3252:                else {
                   3253:                        mp->start = s;
                   3254:                        mp->size = e - s;
                   3255:                }
                   3256:                DPRINTFN(BOOT,
1.85      matt     3257:                    "pmap_bootstrap: a-avail[%d] start %#" _PRIxpa " size %#" _PRIxpa "\n",
                   3258:                    i, mp->start, mp->size);
1.1       matt     3259:        }
                   3260:
                   3261:        /*
                   3262:         * Move (and uncount) all the null return to the end.
                   3263:         */
                   3264:        for (mp = avail, i = 0; i < avail_cnt; i++, mp++) {
                   3265:                if (mp->size == 0) {
                   3266:                        tmp = avail[i];
                   3267:                        avail[i] = avail[--avail_cnt];
                   3268:                        avail[avail_cnt] = avail[i];
                   3269:                }
                   3270:        }
                   3271:
                   3272:        /*
1.61      skrll    3273:         * (Bubble)sort them into ascending order.
1.1       matt     3274:         */
                   3275:        for (i = 0; i < avail_cnt; i++) {
                   3276:                for (j = i + 1; j < avail_cnt; j++) {
                   3277:                        if (avail[i].start > avail[j].start) {
                   3278:                                tmp = avail[i];
                   3279:                                avail[i] = avail[j];
                   3280:                                avail[j] = tmp;
                   3281:                        }
                   3282:                }
                   3283:        }
                   3284:
                   3285:        /*
                   3286:         * Make sure they don't overlap.
                   3287:         */
                   3288:        for (mp = avail, i = 0; i < avail_cnt - 1; i++, mp++) {
                   3289:                if (mp[0].start + mp[0].size > mp[1].start) {
                   3290:                        mp[0].size = mp[1].start - mp[0].start;
                   3291:                }
                   3292:                DPRINTFN(BOOT,
1.85      matt     3293:                    "pmap_bootstrap: avail[%d] start %#" _PRIxpa " size %#" _PRIxpa "\n",
                   3294:                    i, mp->start, mp->size);
1.1       matt     3295:        }
                   3296:        DPRINTFN(BOOT,
1.85      matt     3297:            "pmap_bootstrap: avail[%d] start %#" _PRIxpa " size %#" _PRIxpa "\n",
                   3298:            i, mp->start, mp->size);
1.1       matt     3299:
                   3300: #ifdef PTEGCOUNT
                   3301:        pmap_pteg_cnt = PTEGCOUNT;
                   3302: #else /* PTEGCOUNT */
1.38      sanjayl  3303:
1.1       matt     3304:        pmap_pteg_cnt = 0x1000;
                   3305:
                   3306:        while (pmap_pteg_cnt < physmem)
                   3307:                pmap_pteg_cnt <<= 1;
                   3308:
                   3309:        pmap_pteg_cnt >>= 1;
                   3310: #endif /* PTEGCOUNT */
                   3311:
1.38      sanjayl  3312: #ifdef DEBUG
1.85      matt     3313:        DPRINTFN(BOOT, "pmap_pteg_cnt: 0x%x\n", pmap_pteg_cnt);
1.38      sanjayl  3314: #endif
                   3315:
1.1       matt     3316:        /*
                   3317:         * Find suitably aligned memory for PTEG hash table.
                   3318:         */
1.2       matt     3319:        size = pmap_pteg_cnt * sizeof(struct pteg);
1.53      garbled  3320:        pmap_pteg_table = (void *)(uintptr_t) pmap_boot_find_memory(size, size, 0);
1.38      sanjayl  3321:
                   3322: #ifdef DEBUG
                   3323:        DPRINTFN(BOOT,
1.85      matt     3324:                "PTEG cnt: 0x%x HTAB size: 0x%08x bytes, address: %p\n", pmap_pteg_cnt, (unsigned int)size, pmap_pteg_table);
1.38      sanjayl  3325: #endif
                   3326:
                   3327:
1.1       matt     3328: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
                   3329:        if ( (uintptr_t) pmap_pteg_table + size > SEGMENT_LENGTH)
1.54      mlelstv  3330:                panic("pmap_bootstrap: pmap_pteg_table end (%p + %#" _PRIxpa ") > 256MB",
1.1       matt     3331:                    pmap_pteg_table, size);
                   3332: #endif
                   3333:
1.32      he       3334:        memset(__UNVOLATILE(pmap_pteg_table), 0,
                   3335:                pmap_pteg_cnt * sizeof(struct pteg));
1.1       matt     3336:        pmap_pteg_mask = pmap_pteg_cnt - 1;
                   3337:
                   3338:        /*
                   3339:         * We cannot do pmap_steal_memory here since UVM hasn't been loaded
                   3340:         * with pages.  So we just steal them before giving them to UVM.
                   3341:         */
                   3342:        size = sizeof(pmap_pvo_table[0]) * pmap_pteg_cnt;
1.53      garbled  3343:        pmap_pvo_table = (void *)(uintptr_t) pmap_boot_find_memory(size, PAGE_SIZE, 0);
1.1       matt     3344: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
                   3345:        if ( (uintptr_t) pmap_pvo_table + size > SEGMENT_LENGTH)
1.54      mlelstv  3346:                panic("pmap_bootstrap: pmap_pvo_table end (%p + %#" _PRIxpa ") > 256MB",
1.1       matt     3347:                    pmap_pvo_table, size);
                   3348: #endif
                   3349:
                   3350:        for (i = 0; i < pmap_pteg_cnt; i++)
                   3351:                TAILQ_INIT(&pmap_pvo_table[i]);
                   3352:
                   3353: #ifndef MSGBUFADDR
                   3354:        /*
                   3355:         * Allocate msgbuf in high memory.
                   3356:         */
1.53      garbled  3357:        msgbuf_paddr = pmap_boot_find_memory(MSGBUFSIZE, PAGE_SIZE, 1);
1.1       matt     3358: #endif
                   3359:
                   3360:        for (mp = avail, i = 0; i < avail_cnt; mp++, i++) {
                   3361:                paddr_t pfstart = atop(mp->start);
                   3362:                paddr_t pfend = atop(mp->start + mp->size);
                   3363:                if (mp->size == 0)
                   3364:                        continue;
                   3365:                if (mp->start + mp->size <= SEGMENT_LENGTH) {
                   3366:                        uvm_page_physload(pfstart, pfend, pfstart, pfend,
                   3367:                                VM_FREELIST_FIRST256);
                   3368:                } else if (mp->start >= SEGMENT_LENGTH) {
                   3369:                        uvm_page_physload(pfstart, pfend, pfstart, pfend,
                   3370:                                VM_FREELIST_DEFAULT);
                   3371:                } else {
                   3372:                        pfend = atop(SEGMENT_LENGTH);
                   3373:                        uvm_page_physload(pfstart, pfend, pfstart, pfend,
                   3374:                                VM_FREELIST_FIRST256);
                   3375:                        pfstart = atop(SEGMENT_LENGTH);
                   3376:                        pfend = atop(mp->start + mp->size);
                   3377:                        uvm_page_physload(pfstart, pfend, pfstart, pfend,
                   3378:                                VM_FREELIST_DEFAULT);
                   3379:                }
                   3380:        }
                   3381:
                   3382:        /*
                   3383:         * Make sure kernel vsid is allocated as well as VSID 0.
                   3384:         */
                   3385:        pmap_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS-1)) / VSID_NBPW]
                   3386:                |= 1 << (KERNEL_VSIDBITS % VSID_NBPW);
1.53      garbled  3387:        pmap_vsid_bitmap[(PHYSMAP_VSIDBITS & (NPMAPS-1)) / VSID_NBPW]
                   3388:                |= 1 << (PHYSMAP_VSIDBITS % VSID_NBPW);
1.1       matt     3389:        pmap_vsid_bitmap[0] |= 1;
                   3390:
                   3391:        /*
1.103     thorpej  3392:         * Initialize kernel pmap.
1.1       matt     3393:         */
1.103     thorpej  3394: #if defined(PMAP_OEA) || defined(PMAP_OEA64_BRIDGE)
1.1       matt     3395:        for (i = 0; i < 16; i++) {
1.38      sanjayl  3396:                pmap_kernel()->pm_sr[i] = KERNELN_SEGMENT(i)|SR_PRKEY;
1.1       matt     3397:        }
1.102     thorpej  3398:        pmap_kernel()->pm_vsid = KERNEL_VSIDBITS;
1.1       matt     3399:
                   3400:        pmap_kernel()->pm_sr[KERNEL_SR] = KERNEL_SEGMENT|SR_SUKEY|SR_PRKEY;
                   3401: #ifdef KERNEL2_SR
                   3402:        pmap_kernel()->pm_sr[KERNEL2_SR] = KERNEL2_SEGMENT|SR_SUKEY|SR_PRKEY;
                   3403: #endif
1.53      garbled  3404: #endif /* PMAP_OEA || PMAP_OEA64_BRIDGE */
1.103     thorpej  3405:
                   3406: #if defined(PMAP_OEA) && defined(PPC_OEA601)
1.105     thorpej  3407:        if ((MFPVR() >> 16) == MPC601) {
1.103     thorpej  3408:                for (i = 0; i < 16; i++) {
                   3409:                        if (iosrtable[i] & SR601_T) {
                   3410:                                pmap_kernel()->pm_sr[i] = iosrtable[i];
                   3411:                        }
1.1       matt     3412:                }
                   3413:        }
1.103     thorpej  3414: #endif /* PMAP_OEA && PPC_OEA601 */
1.1       matt     3415:
                   3416: #ifdef ALTIVEC
                   3417:        pmap_use_altivec = cpu_altivec;
                   3418: #endif
                   3419:
                   3420: #ifdef DEBUG
                   3421:        if (pmapdebug & PMAPDEBUG_BOOT) {
                   3422:                u_int cnt;
1.94      cherry   3423:                uvm_physseg_t bank;
1.1       matt     3424:                char pbuf[9];
1.94      cherry   3425:                for (cnt = 0, bank = uvm_physseg_get_first();
                   3426:                     uvm_physseg_valid_p(bank);
                   3427:                     bank = uvm_physseg_get_next(bank)) {
                   3428:                        cnt += uvm_physseg_get_avail_end(bank) -
                   3429:                            uvm_physseg_get_avail_start(bank);
1.53      garbled  3430:                        printf("pmap_bootstrap: vm_physmem[%d]=%#" _PRIxpa "-%#" _PRIxpa "/%#" _PRIxpa "\n",
1.1       matt     3431:                            bank,
1.94      cherry   3432:                            ptoa(uvm_physseg_get_avail_start(bank)),
                   3433:                            ptoa(uvm_physseg_get_avail_end(bank)),
                   3434:                            ptoa(uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank)));
1.1       matt     3435:                }
                   3436:                format_bytes(pbuf, sizeof(pbuf), ptoa((u_int64_t) cnt));
                   3437:                printf("pmap_bootstrap: UVM memory = %s (%u pages)\n",
                   3438:                    pbuf, cnt);
                   3439:        }
                   3440: #endif
                   3441:
1.106     martin   3442:        pool_init(&pmap_pvo_pool, sizeof(struct pvo_entry),
                   3443:            sizeof(struct pvo_entry), 0, 0, "pmap_pvopl",
                   3444:            &pmap_pool_allocator, IPL_VM);
1.1       matt     3445:
1.106     martin   3446:        pool_setlowat(&pmap_pvo_pool, 1008);
1.1       matt     3447:
                   3448:        pool_init(&pmap_pool, sizeof(struct pmap),
1.106     martin   3449:            sizeof(void *), 0, 0, "pmap_pl", &pmap_pool_allocator,
1.48      ad       3450:            IPL_NONE);
1.41      matt     3451:
1.89      macallan 3452: #if defined(PMAP_NEED_MAPKERNEL)
1.41      matt     3453:        {
1.53      garbled  3454:                struct pmap *pm = pmap_kernel();
1.58      garbled  3455: #if defined(PMAP_NEED_FULL_MAPKERNEL)
1.41      matt     3456:                extern int etext[], kernel_text[];
                   3457:                vaddr_t va, va_etext = (paddr_t) etext;
1.53      garbled  3458: #endif
                   3459:                paddr_t pa, pa_end;
1.42      matt     3460:                register_t sr;
1.53      garbled  3461:                struct pte pt;
                   3462:                unsigned int ptegidx;
                   3463:                int bank;
1.42      matt     3464:
1.53      garbled  3465:                sr = PHYSMAPN_SEGMENT(0) | SR_SUKEY|SR_PRKEY;
                   3466:                pm->pm_sr[0] = sr;
1.41      matt     3467:
1.53      garbled  3468:                for (bank = 0; bank < vm_nphysseg; bank++) {
1.73      uebayasi 3469:                        pa_end = ptoa(VM_PHYSMEM_PTR(bank)->avail_end);
                   3470:                        pa = ptoa(VM_PHYSMEM_PTR(bank)->avail_start);
1.53      garbled  3471:                        for (; pa < pa_end; pa += PAGE_SIZE) {
                   3472:                                ptegidx = va_to_pteg(pm, pa);
                   3473:                                pmap_pte_create(&pt, pm, pa, pa | PTE_M|PTE_BW);
                   3474:                                pmap_pte_insert(ptegidx, &pt);
                   3475:                        }
                   3476:                }
                   3477:
1.58      garbled  3478: #if defined(PMAP_NEED_FULL_MAPKERNEL)
1.41      matt     3479:                va = (vaddr_t) kernel_text;
                   3480:
                   3481:                for (pa = kernelstart; va < va_etext;
1.53      garbled  3482:                     pa += PAGE_SIZE, va += PAGE_SIZE) {
                   3483:                        ptegidx = va_to_pteg(pm, va);
                   3484:                        pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BR);
                   3485:                        pmap_pte_insert(ptegidx, &pt);
                   3486:                }
1.41      matt     3487:
                   3488:                for (; pa < kernelend;
1.53      garbled  3489:                     pa += PAGE_SIZE, va += PAGE_SIZE) {
                   3490:                        ptegidx = va_to_pteg(pm, va);
                   3491:                        pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BW);
                   3492:                        pmap_pte_insert(ptegidx, &pt);
                   3493:                }
                   3494:
1.58      garbled  3495:                for (va = 0, pa = 0; va < kernelstart;
1.53      garbled  3496:                     pa += PAGE_SIZE, va += PAGE_SIZE) {
                   3497:                        ptegidx = va_to_pteg(pm, va);
1.58      garbled  3498:                        if (va < 0x3000)
                   3499:                                pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BR);
                   3500:                        else
                   3501:                                pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BW);
                   3502:                        pmap_pte_insert(ptegidx, &pt);
                   3503:                }
                   3504:                for (va = kernelend, pa = kernelend; va < SEGMENT_LENGTH;
                   3505:                    pa += PAGE_SIZE, va += PAGE_SIZE) {
                   3506:                        ptegidx = va_to_pteg(pm, va);
1.53      garbled  3507:                        pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BW);
                   3508:                        pmap_pte_insert(ptegidx, &pt);
                   3509:                }
1.103     thorpej  3510: #endif /* PMAP_NEED_FULL_MAPKERNEL */
1.99      thorpej  3511:        }
1.103     thorpej  3512: #endif /* PMAP_NEED_MAPKERNEL */
1.99      thorpej  3513: }
1.42      matt     3514:
1.99      thorpej  3515: /*
                   3516:  * Using the data structures prepared in pmap_bootstrap1(), program
                   3517:  * the MMU hardware.
                   3518:  */
                   3519: void
                   3520: pmap_bootstrap2(void)
                   3521: {
1.103     thorpej  3522: #if defined(PMAP_OEA) || defined(PMAP_OEA64_BRIDGE)
1.99      thorpej  3523:        for (int i = 0; i < 16; i++) {
                   3524:                __asm volatile("mtsrin %0,%1"
                   3525:                        :: "r"(pmap_kernel()->pm_sr[i]),
                   3526:                           "r"(i << ADDR_SR_SHFT));
1.41      matt     3527:        }
1.99      thorpej  3528: #endif /* PMAP_OEA || PMAP_OEA64_BRIDGE */
1.103     thorpej  3529:
                   3530: #if defined(PMAP_OEA)
1.109     riastrad 3531:        __asm volatile("sync; mtsdr1 %0; isync"
                   3532:            :
                   3533:            : "r"((uintptr_t)pmap_pteg_table | (pmap_pteg_mask >> 10))
                   3534:            : "memory");
1.103     thorpej  3535: #elif defined(PMAP_OEA64) || defined(PMAP_OEA64_BRIDGE)
1.109     riastrad 3536:        __asm volatile("sync; mtsdr1 %0; isync"
                   3537:            :
                   3538:            : "r"((uintptr_t)pmap_pteg_table |
                   3539:                (32 - __builtin_clz(pmap_pteg_mask >> 11)))
                   3540:            : "memory");
1.41      matt     3541: #endif
1.99      thorpej  3542:        tlbia();
1.91      macallan 3543:
                   3544: #if defined(PMAPDEBUG)
1.103     thorpej  3545:        if (pmapdebug)
1.91      macallan 3546:            pmap_print_mmuregs();
                   3547: #endif
1.1       matt     3548: }
1.99      thorpej  3549:
                   3550: /*
                   3551:  * This is not part of the defined PMAP interface and is specific to the
                   3552:  * PowerPC architecture.  This is called during initppc, before the system
                   3553:  * is really initialized.
                   3554:  */
                   3555: void
                   3556: pmap_bootstrap(paddr_t kernelstart, paddr_t kernelend)
                   3557: {
                   3558:        pmap_bootstrap1(kernelstart, kernelend);
                   3559:        pmap_bootstrap2();
                   3560: }

CVSweb <webmaster@jp.NetBSD.org>