Annotation of src/sys/arch/powerpc/oea/pmap.c, Revision 1.111
1.111 ! martin 1: /* $NetBSD: pmap.c,v 1.110 2022/02/18 18:58:15 martin Exp $ */
1.1 matt 2: /*-
3: * Copyright (c) 2001 The NetBSD Foundation, Inc.
4: * All rights reserved.
5: *
6: * This code is derived from software contributed to The NetBSD Foundation
7: * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc.
8: *
1.38 sanjayl 9: * Support for PPC64 Bridge mode added by Sanjay Lal <sanjayl@kymasys.com>
10: * of Kyma Systems LLC.
11: *
1.1 matt 12: * Redistribution and use in source and binary forms, with or without
13: * modification, are permitted provided that the following conditions
14: * are met:
15: * 1. Redistributions of source code must retain the above copyright
16: * notice, this list of conditions and the following disclaimer.
17: * 2. Redistributions in binary form must reproduce the above copyright
18: * notice, this list of conditions and the following disclaimer in the
19: * documentation and/or other materials provided with the distribution.
20: *
21: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
22: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
25: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31: * POSSIBILITY OF SUCH DAMAGE.
32: */
33:
34: /*
35: * Copyright (C) 1995, 1996 Wolfgang Solfrank.
36: * Copyright (C) 1995, 1996 TooLs GmbH.
37: * All rights reserved.
38: *
39: * Redistribution and use in source and binary forms, with or without
40: * modification, are permitted provided that the following conditions
41: * are met:
42: * 1. Redistributions of source code must retain the above copyright
43: * notice, this list of conditions and the following disclaimer.
44: * 2. Redistributions in binary form must reproduce the above copyright
45: * notice, this list of conditions and the following disclaimer in the
46: * documentation and/or other materials provided with the distribution.
47: * 3. All advertising materials mentioning features or use of this software
48: * must display the following acknowledgement:
49: * This product includes software developed by TooLs GmbH.
50: * 4. The name of TooLs GmbH may not be used to endorse or promote products
51: * derived from this software without specific prior written permission.
52: *
53: * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
54: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
55: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
56: * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
57: * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
58: * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
59: * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
60: * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
61: * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
62: * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
63: */
1.11 lukem 64:
65: #include <sys/cdefs.h>
1.111 ! martin 66: __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.110 2022/02/18 18:58:15 martin Exp $");
1.53 garbled 67:
68: #define PMAP_NOOPNAMES
1.1 matt 69:
1.98 rin 70: #ifdef _KERNEL_OPT
1.1 matt 71: #include "opt_altivec.h"
1.57 matt 72: #include "opt_multiprocessor.h"
1.1 matt 73: #include "opt_pmap.h"
1.98 rin 74: #include "opt_ppcarch.h"
75: #endif
1.57 matt 76:
1.1 matt 77: #include <sys/param.h>
78: #include <sys/proc.h>
79: #include <sys/pool.h>
80: #include <sys/queue.h>
81: #include <sys/device.h> /* for evcnt */
82: #include <sys/systm.h>
1.50 ad 83: #include <sys/atomic.h>
1.1 matt 84:
85: #include <uvm/uvm.h>
1.94 cherry 86: #include <uvm/uvm_physseg.h>
1.1 matt 87:
88: #include <machine/powerpc.h>
1.80 matt 89: #include <powerpc/bat.h>
90: #include <powerpc/pcb.h>
91: #include <powerpc/psl.h>
1.1 matt 92: #include <powerpc/spr.h>
1.71 matt 93: #include <powerpc/oea/spr.h>
94: #include <powerpc/oea/sr_601.h>
1.1 matt 95:
96: #ifdef ALTIVEC
1.86 matt 97: extern int pmap_use_altivec;
1.1 matt 98: #endif
99:
1.21 aymeric 100: #ifdef PMAP_MEMLIMIT
1.53 garbled 101: static paddr_t pmap_memlimit = PMAP_MEMLIMIT;
1.21 aymeric 102: #else
1.53 garbled 103: static paddr_t pmap_memlimit = -PAGE_SIZE; /* there is no limit */
1.21 aymeric 104: #endif
1.1 matt 105:
1.86 matt 106: extern struct pmap kernel_pmap_;
107: static unsigned int pmap_pages_stolen;
108: static u_long pmap_pte_valid;
1.1 matt 109: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1.86 matt 110: static u_long pmap_pvo_enter_depth;
111: static u_long pmap_pvo_remove_depth;
1.1 matt 112: #endif
113:
114: #ifndef MSGBUFADDR
115: extern paddr_t msgbuf_paddr;
116: #endif
117:
118: static struct mem_region *mem, *avail;
119: static u_int mem_cnt, avail_cnt;
120:
1.53 garbled 121: #if !defined(PMAP_OEA64) && !defined(PMAP_OEA64_BRIDGE)
122: # define PMAP_OEA 1
123: #endif
124:
125: #if defined(PMAP_OEA)
126: #define _PRIxpte "lx"
127: #else
128: #define _PRIxpte PRIx64
129: #endif
130: #define _PRIxpa "lx"
131: #define _PRIxva "lx"
1.54 mlelstv 132: #define _PRIsr "lx"
1.53 garbled 133:
1.76 matt 134: #ifdef PMAP_NEEDS_FIXUP
1.53 garbled 135: #if defined(PMAP_OEA)
136: #define PMAPNAME(name) pmap32_##name
137: #elif defined(PMAP_OEA64)
138: #define PMAPNAME(name) pmap64_##name
139: #elif defined(PMAP_OEA64_BRIDGE)
140: #define PMAPNAME(name) pmap64bridge_##name
141: #else
142: #error unknown variant for pmap
143: #endif
1.76 matt 144: #endif /* PMAP_NEEDS_FIXUP */
1.53 garbled 145:
1.76 matt 146: #ifdef PMAPNAME
1.53 garbled 147: #define STATIC static
148: #define pmap_pte_spill PMAPNAME(pte_spill)
149: #define pmap_real_memory PMAPNAME(real_memory)
150: #define pmap_init PMAPNAME(init)
151: #define pmap_virtual_space PMAPNAME(virtual_space)
152: #define pmap_create PMAPNAME(create)
153: #define pmap_reference PMAPNAME(reference)
154: #define pmap_destroy PMAPNAME(destroy)
155: #define pmap_copy PMAPNAME(copy)
156: #define pmap_update PMAPNAME(update)
157: #define pmap_enter PMAPNAME(enter)
158: #define pmap_remove PMAPNAME(remove)
159: #define pmap_kenter_pa PMAPNAME(kenter_pa)
160: #define pmap_kremove PMAPNAME(kremove)
161: #define pmap_extract PMAPNAME(extract)
162: #define pmap_protect PMAPNAME(protect)
163: #define pmap_unwire PMAPNAME(unwire)
164: #define pmap_page_protect PMAPNAME(page_protect)
1.111 ! martin 165: #define pmap_pv_protect PMAPNAME(pv_protect)
1.53 garbled 166: #define pmap_query_bit PMAPNAME(query_bit)
167: #define pmap_clear_bit PMAPNAME(clear_bit)
168:
169: #define pmap_activate PMAPNAME(activate)
170: #define pmap_deactivate PMAPNAME(deactivate)
171:
172: #define pmap_pinit PMAPNAME(pinit)
173: #define pmap_procwr PMAPNAME(procwr)
174:
1.86 matt 175: #define pmap_pool PMAPNAME(pool)
1.106 martin 176: #define pmap_pvo_pool PMAPNAME(pvo_pool)
1.86 matt 177: #define pmap_pvo_table PMAPNAME(pvo_table)
1.53 garbled 178: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
179: #define pmap_pte_print PMAPNAME(pte_print)
180: #define pmap_pteg_check PMAPNAME(pteg_check)
181: #define pmap_print_mmruregs PMAPNAME(print_mmuregs)
182: #define pmap_print_pte PMAPNAME(print_pte)
183: #define pmap_pteg_dist PMAPNAME(pteg_dist)
184: #endif
185: #if defined(DEBUG) || defined(PMAPCHECK)
186: #define pmap_pvo_verify PMAPNAME(pvo_verify)
1.56 phx 187: #define pmapcheck PMAPNAME(check)
188: #endif
189: #if defined(DEBUG) || defined(PMAPDEBUG)
190: #define pmapdebug PMAPNAME(debug)
1.53 garbled 191: #endif
192: #define pmap_steal_memory PMAPNAME(steal_memory)
193: #define pmap_bootstrap PMAPNAME(bootstrap)
1.100 rin 194: #define pmap_bootstrap1 PMAPNAME(bootstrap1)
195: #define pmap_bootstrap2 PMAPNAME(bootstrap2)
1.53 garbled 196: #else
197: #define STATIC /* nothing */
198: #endif /* PMAPNAME */
199:
200: STATIC int pmap_pte_spill(struct pmap *, vaddr_t, bool);
201: STATIC void pmap_real_memory(paddr_t *, psize_t *);
202: STATIC void pmap_init(void);
203: STATIC void pmap_virtual_space(vaddr_t *, vaddr_t *);
204: STATIC pmap_t pmap_create(void);
205: STATIC void pmap_reference(pmap_t);
206: STATIC void pmap_destroy(pmap_t);
207: STATIC void pmap_copy(pmap_t, pmap_t, vaddr_t, vsize_t, vaddr_t);
208: STATIC void pmap_update(pmap_t);
1.65 cegger 209: STATIC int pmap_enter(pmap_t, vaddr_t, paddr_t, vm_prot_t, u_int);
1.53 garbled 210: STATIC void pmap_remove(pmap_t, vaddr_t, vaddr_t);
1.68 cegger 211: STATIC void pmap_kenter_pa(vaddr_t, paddr_t, vm_prot_t, u_int);
1.53 garbled 212: STATIC void pmap_kremove(vaddr_t, vsize_t);
213: STATIC bool pmap_extract(pmap_t, vaddr_t, paddr_t *);
214:
215: STATIC void pmap_protect(pmap_t, vaddr_t, vaddr_t, vm_prot_t);
216: STATIC void pmap_unwire(pmap_t, vaddr_t);
217: STATIC void pmap_page_protect(struct vm_page *, vm_prot_t);
1.111 ! martin 218: STATIC void pmap_pv_protect(paddr_t, vm_prot_t);
1.53 garbled 219: STATIC bool pmap_query_bit(struct vm_page *, int);
220: STATIC bool pmap_clear_bit(struct vm_page *, int);
221:
222: STATIC void pmap_activate(struct lwp *);
223: STATIC void pmap_deactivate(struct lwp *);
224:
225: STATIC void pmap_pinit(pmap_t pm);
226: STATIC void pmap_procwr(struct proc *, vaddr_t, size_t);
227:
228: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
229: STATIC void pmap_pte_print(volatile struct pte *);
230: STATIC void pmap_pteg_check(void);
231: STATIC void pmap_print_mmuregs(void);
232: STATIC void pmap_print_pte(pmap_t, vaddr_t);
233: STATIC void pmap_pteg_dist(void);
234: #endif
235: #if defined(DEBUG) || defined(PMAPCHECK)
236: STATIC void pmap_pvo_verify(void);
237: #endif
238: STATIC vaddr_t pmap_steal_memory(vsize_t, vaddr_t *, vaddr_t *);
239: STATIC void pmap_bootstrap(paddr_t, paddr_t);
1.100 rin 240: STATIC void pmap_bootstrap1(paddr_t, paddr_t);
241: STATIC void pmap_bootstrap2(void);
1.53 garbled 242:
243: #ifdef PMAPNAME
244: const struct pmap_ops PMAPNAME(ops) = {
245: .pmapop_pte_spill = pmap_pte_spill,
246: .pmapop_real_memory = pmap_real_memory,
247: .pmapop_init = pmap_init,
248: .pmapop_virtual_space = pmap_virtual_space,
249: .pmapop_create = pmap_create,
250: .pmapop_reference = pmap_reference,
251: .pmapop_destroy = pmap_destroy,
252: .pmapop_copy = pmap_copy,
253: .pmapop_update = pmap_update,
254: .pmapop_enter = pmap_enter,
255: .pmapop_remove = pmap_remove,
256: .pmapop_kenter_pa = pmap_kenter_pa,
257: .pmapop_kremove = pmap_kremove,
258: .pmapop_extract = pmap_extract,
259: .pmapop_protect = pmap_protect,
260: .pmapop_unwire = pmap_unwire,
261: .pmapop_page_protect = pmap_page_protect,
1.111 ! martin 262: .pmapop_pv_protect = pmap_pv_protect,
1.53 garbled 263: .pmapop_query_bit = pmap_query_bit,
264: .pmapop_clear_bit = pmap_clear_bit,
265: .pmapop_activate = pmap_activate,
266: .pmapop_deactivate = pmap_deactivate,
267: .pmapop_pinit = pmap_pinit,
268: .pmapop_procwr = pmap_procwr,
269: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
270: .pmapop_pte_print = pmap_pte_print,
271: .pmapop_pteg_check = pmap_pteg_check,
272: .pmapop_print_mmuregs = pmap_print_mmuregs,
273: .pmapop_print_pte = pmap_print_pte,
274: .pmapop_pteg_dist = pmap_pteg_dist,
275: #else
276: .pmapop_pte_print = NULL,
277: .pmapop_pteg_check = NULL,
278: .pmapop_print_mmuregs = NULL,
279: .pmapop_print_pte = NULL,
280: .pmapop_pteg_dist = NULL,
281: #endif
282: #if defined(DEBUG) || defined(PMAPCHECK)
283: .pmapop_pvo_verify = pmap_pvo_verify,
284: #else
285: .pmapop_pvo_verify = NULL,
1.1 matt 286: #endif
1.53 garbled 287: .pmapop_steal_memory = pmap_steal_memory,
288: .pmapop_bootstrap = pmap_bootstrap,
1.101 thorpej 289: .pmapop_bootstrap1 = pmap_bootstrap1,
290: .pmapop_bootstrap2 = pmap_bootstrap2,
1.53 garbled 291: };
292: #endif /* !PMAPNAME */
1.1 matt 293:
294: /*
1.38 sanjayl 295: * The following structure is aligned to 32 bytes
1.1 matt 296: */
297: struct pvo_entry {
298: LIST_ENTRY(pvo_entry) pvo_vlink; /* Link to common virt page */
299: TAILQ_ENTRY(pvo_entry) pvo_olink; /* Link to overflow entry */
300: struct pte pvo_pte; /* Prebuilt PTE */
301: pmap_t pvo_pmap; /* ptr to owning pmap */
302: vaddr_t pvo_vaddr; /* VA of entry */
303: #define PVO_PTEGIDX_MASK 0x0007 /* which PTEG slot */
304: #define PVO_PTEGIDX_VALID 0x0008 /* slot is valid */
305: #define PVO_WIRED 0x0010 /* PVO entry is wired */
306: #define PVO_MANAGED 0x0020 /* PVO e. for managed page */
307: #define PVO_EXECUTABLE 0x0040 /* PVO e. for executable page */
1.39 matt 308: #define PVO_WIRED_P(pvo) ((pvo)->pvo_vaddr & PVO_WIRED)
309: #define PVO_MANAGED_P(pvo) ((pvo)->pvo_vaddr & PVO_MANAGED)
310: #define PVO_EXECUTABLE_P(pvo) ((pvo)->pvo_vaddr & PVO_EXECUTABLE)
1.12 matt 311: #define PVO_ENTER_INSERT 0 /* PVO has been removed */
312: #define PVO_SPILL_UNSET 1 /* PVO has been evicted */
313: #define PVO_SPILL_SET 2 /* PVO has been spilled */
314: #define PVO_SPILL_INSERT 3 /* PVO has been inserted */
315: #define PVO_PMAP_PAGE_PROTECT 4 /* PVO has changed */
316: #define PVO_PMAP_PROTECT 5 /* PVO has changed */
317: #define PVO_REMOVE 6 /* PVO has been removed */
318: #define PVO_WHERE_MASK 15
319: #define PVO_WHERE_SHFT 8
1.38 sanjayl 320: } __attribute__ ((aligned (32)));
1.1 matt 321: #define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF)
322: #define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
323: #define PVO_PTEGIDX_ISSET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID)
324: #define PVO_PTEGIDX_CLR(pvo) \
325: ((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK)))
326: #define PVO_PTEGIDX_SET(pvo,i) \
327: ((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID))
1.12 matt 328: #define PVO_WHERE(pvo,w) \
329: ((pvo)->pvo_vaddr &= ~(PVO_WHERE_MASK << PVO_WHERE_SHFT), \
330: (pvo)->pvo_vaddr |= ((PVO_ ## w) << PVO_WHERE_SHFT))
1.1 matt 331:
332: TAILQ_HEAD(pvo_tqhead, pvo_entry);
333: struct pvo_tqhead *pmap_pvo_table; /* pvo entries by ptegroup index */
334:
335: struct pool pmap_pool; /* pool for pmap structures */
1.106 martin 336: struct pool pmap_pvo_pool; /* pool for pvo entries */
1.1 matt 337:
338: /*
339: * We keep a cache of unmanaged pages to be used for pvo entries for
340: * unmanaged pages.
341: */
342: struct pvo_page {
343: SIMPLEQ_ENTRY(pvo_page) pvop_link;
344: };
345: SIMPLEQ_HEAD(pvop_head, pvo_page);
1.106 martin 346: static struct pvop_head pmap_pvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_pvop_head);
347: static u_long pmap_pvop_free;
348: static u_long pmap_pvop_maxfree;
349:
350: static void *pmap_pool_alloc(struct pool *, int);
351: static void pmap_pool_free(struct pool *, void *);
352:
353: static struct pool_allocator pmap_pool_allocator = {
354: .pa_alloc = pmap_pool_alloc,
355: .pa_free = pmap_pool_free,
1.43 garbled 356: .pa_pagesz = 0,
1.1 matt 357: };
358:
359: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
1.2 matt 360: void pmap_pte_print(volatile struct pte *);
1.1 matt 361: void pmap_pteg_check(void);
362: void pmap_pteg_dist(void);
363: void pmap_print_pte(pmap_t, vaddr_t);
364: void pmap_print_mmuregs(void);
365: #endif
366:
367: #if defined(DEBUG) || defined(PMAPCHECK)
368: #ifdef PMAPCHECK
369: int pmapcheck = 1;
370: #else
371: int pmapcheck = 0;
372: #endif
373: void pmap_pvo_verify(void);
1.53 garbled 374: static void pmap_pvo_check(const struct pvo_entry *);
1.1 matt 375: #define PMAP_PVO_CHECK(pvo) \
376: do { \
377: if (pmapcheck) \
378: pmap_pvo_check(pvo); \
379: } while (0)
380: #else
381: #define PMAP_PVO_CHECK(pvo) do { } while (/*CONSTCOND*/0)
382: #endif
1.53 garbled 383: static int pmap_pte_insert(int, struct pte *);
384: static int pmap_pvo_enter(pmap_t, struct pool *, struct pvo_head *,
1.2 matt 385: vaddr_t, paddr_t, register_t, int);
1.53 garbled 386: static void pmap_pvo_remove(struct pvo_entry *, int, struct pvo_head *);
387: static void pmap_pvo_free(struct pvo_entry *);
388: static void pmap_pvo_free_list(struct pvo_head *);
389: static struct pvo_entry *pmap_pvo_find_va(pmap_t, vaddr_t, int *);
390: static volatile struct pte *pmap_pvo_to_pte(const struct pvo_entry *, int);
391: static struct pvo_entry *pmap_pvo_reclaim(struct pmap *);
392: static void pvo_set_exec(struct pvo_entry *);
393: static void pvo_clear_exec(struct pvo_entry *);
1.1 matt 394:
1.53 garbled 395: static void tlbia(void);
1.1 matt 396:
1.53 garbled 397: static void pmap_release(pmap_t);
398: static paddr_t pmap_boot_find_memory(psize_t, psize_t, int);
1.1 matt 399:
1.25 chs 400: static uint32_t pmap_pvo_reclaim_nextidx;
401: #ifdef DEBUG
402: static int pmap_pvo_reclaim_debugctr;
403: #endif
404:
1.1 matt 405: #define VSID_NBPW (sizeof(uint32_t) * 8)
406: static uint32_t pmap_vsid_bitmap[NPMAPS / VSID_NBPW];
407:
408: static int pmap_initialized;
409:
410: #if defined(DEBUG) || defined(PMAPDEBUG)
411: #define PMAPDEBUG_BOOT 0x0001
412: #define PMAPDEBUG_PTE 0x0002
413: #define PMAPDEBUG_EXEC 0x0008
414: #define PMAPDEBUG_PVOENTER 0x0010
415: #define PMAPDEBUG_PVOREMOVE 0x0020
416: #define PMAPDEBUG_ACTIVATE 0x0100
417: #define PMAPDEBUG_CREATE 0x0200
418: #define PMAPDEBUG_ENTER 0x1000
419: #define PMAPDEBUG_KENTER 0x2000
420: #define PMAPDEBUG_KREMOVE 0x4000
421: #define PMAPDEBUG_REMOVE 0x8000
1.38 sanjayl 422:
1.1 matt 423: unsigned int pmapdebug = 0;
1.38 sanjayl 424:
1.85 matt 425: # define DPRINTF(x, ...) printf(x, __VA_ARGS__)
426: # define DPRINTFN(n, x, ...) do if (pmapdebug & PMAPDEBUG_ ## n) printf(x, __VA_ARGS__); while (0)
1.1 matt 427: #else
1.85 matt 428: # define DPRINTF(x, ...) do { } while (0)
429: # define DPRINTFN(n, x, ...) do { } while (0)
1.1 matt 430: #endif
431:
432:
433: #ifdef PMAPCOUNTERS
434: /*
435: * From pmap_subr.c
436: */
1.53 garbled 437: extern struct evcnt pmap_evcnt_mappings;
438: extern struct evcnt pmap_evcnt_unmappings;
439:
440: extern struct evcnt pmap_evcnt_kernel_mappings;
441: extern struct evcnt pmap_evcnt_kernel_unmappings;
442:
443: extern struct evcnt pmap_evcnt_mappings_replaced;
444:
445: extern struct evcnt pmap_evcnt_exec_mappings;
446: extern struct evcnt pmap_evcnt_exec_cached;
447:
448: extern struct evcnt pmap_evcnt_exec_synced;
449: extern struct evcnt pmap_evcnt_exec_synced_clear_modify;
450: extern struct evcnt pmap_evcnt_exec_synced_pvo_remove;
451:
452: extern struct evcnt pmap_evcnt_exec_uncached_page_protect;
453: extern struct evcnt pmap_evcnt_exec_uncached_clear_modify;
454: extern struct evcnt pmap_evcnt_exec_uncached_zero_page;
455: extern struct evcnt pmap_evcnt_exec_uncached_copy_page;
456: extern struct evcnt pmap_evcnt_exec_uncached_pvo_remove;
457:
458: extern struct evcnt pmap_evcnt_updates;
459: extern struct evcnt pmap_evcnt_collects;
460: extern struct evcnt pmap_evcnt_copies;
461:
462: extern struct evcnt pmap_evcnt_ptes_spilled;
463: extern struct evcnt pmap_evcnt_ptes_unspilled;
464: extern struct evcnt pmap_evcnt_ptes_evicted;
465:
466: extern struct evcnt pmap_evcnt_ptes_primary[8];
467: extern struct evcnt pmap_evcnt_ptes_secondary[8];
468: extern struct evcnt pmap_evcnt_ptes_removed;
469: extern struct evcnt pmap_evcnt_ptes_changed;
470: extern struct evcnt pmap_evcnt_pvos_reclaimed;
471: extern struct evcnt pmap_evcnt_pvos_failed;
472:
1.1 matt 473: extern struct evcnt pmap_evcnt_zeroed_pages;
474: extern struct evcnt pmap_evcnt_copied_pages;
475: extern struct evcnt pmap_evcnt_idlezeroed_pages;
1.26 matt 476:
1.53 garbled 477: #define PMAPCOUNT(ev) ((pmap_evcnt_ ## ev).ev_count++)
478: #define PMAPCOUNT2(ev) ((ev).ev_count++)
1.1 matt 479: #else
480: #define PMAPCOUNT(ev) ((void) 0)
481: #define PMAPCOUNT2(ev) ((void) 0)
482: #endif
483:
1.109 riastrad 484: #define TLBIE(va) __asm volatile("tlbie %0" :: "r"(va) : "memory")
1.38 sanjayl 485:
486: /* XXXSL: this needs to be moved to assembler */
1.109 riastrad 487: #define TLBIEL(va) __asm volatile("tlbie %0" :: "r"(va) : "memory")
1.38 sanjayl 488:
1.87 kiyohara 489: #ifdef MD_TLBSYNC
490: #define TLBSYNC() MD_TLBSYNC()
491: #else
1.109 riastrad 492: #define TLBSYNC() __asm volatile("tlbsync" ::: "memory")
1.87 kiyohara 493: #endif
1.109 riastrad 494: #define SYNC() __asm volatile("sync" ::: "memory")
495: #define EIEIO() __asm volatile("eieio" ::: "memory")
496: #define DCBST(va) __asm volatile("dcbst 0,%0" :: "r"(va) : "memory")
1.1 matt 497: #define MFMSR() mfmsr()
498: #define MTMSR(psl) mtmsr(psl)
499: #define MFPVR() mfpvr()
500: #define MFSRIN(va) mfsrin(va)
501: #define MFTB() mfrtcltbl()
502:
1.92 joerg 503: #if defined(DDB) && !defined(PMAP_OEA64)
1.35 perry 504: static inline register_t
1.1 matt 505: mfsrin(vaddr_t va)
506: {
1.2 matt 507: register_t sr;
1.35 perry 508: __asm volatile ("mfsrin %0,%1" : "=r"(sr) : "r"(va));
1.1 matt 509: return sr;
510: }
1.92 joerg 511: #endif /* DDB && !PMAP_OEA64 */
1.38 sanjayl 512:
1.53 garbled 513: #if defined (PMAP_OEA64_BRIDGE)
1.38 sanjayl 514: extern void mfmsr64 (register64_t *result);
1.53 garbled 515: #endif /* PMAP_OEA64_BRIDGE */
1.38 sanjayl 516:
1.50 ad 517: #define PMAP_LOCK() KERNEL_LOCK(1, NULL)
518: #define PMAP_UNLOCK() KERNEL_UNLOCK_ONE(NULL)
1.1 matt 519:
1.35 perry 520: static inline register_t
1.1 matt 521: pmap_interrupts_off(void)
522: {
1.2 matt 523: register_t msr = MFMSR();
1.1 matt 524: if (msr & PSL_EE)
525: MTMSR(msr & ~PSL_EE);
526: return msr;
527: }
528:
529: static void
1.2 matt 530: pmap_interrupts_restore(register_t msr)
1.1 matt 531: {
532: if (msr & PSL_EE)
533: MTMSR(msr);
534: }
535:
1.35 perry 536: static inline u_int32_t
1.1 matt 537: mfrtcltbl(void)
538: {
1.55 garbled 539: #ifdef PPC_OEA601
1.1 matt 540: if ((MFPVR() >> 16) == MPC601)
541: return (mfrtcl() >> 7);
542: else
1.55 garbled 543: #endif
1.1 matt 544: return (mftbl());
545: }
546:
547: /*
548: * These small routines may have to be replaced,
549: * if/when we support processors other that the 604.
550: */
551:
552: void
553: tlbia(void)
554: {
1.47 macallan 555: char *i;
1.1 matt 556:
557: SYNC();
1.53 garbled 558: #if defined(PMAP_OEA)
1.1 matt 559: /*
560: * Why not use "tlbia"? Because not all processors implement it.
561: *
1.20 wiz 562: * This needs to be a per-CPU callback to do the appropriate thing
1.1 matt 563: * for the CPU. XXX
564: */
1.47 macallan 565: for (i = 0; i < (char *)0x00040000; i += 0x00001000) {
1.1 matt 566: TLBIE(i);
567: EIEIO();
568: SYNC();
569: }
1.53 garbled 570: #elif defined (PMAP_OEA64) || defined (PMAP_OEA64_BRIDGE)
1.38 sanjayl 571: /* This is specifically for the 970, 970UM v1.6 pp. 140. */
1.51 garbled 572: for (i = 0; i <= (char *)0xFF000; i += 0x00001000) {
1.38 sanjayl 573: TLBIEL(i);
574: EIEIO();
575: SYNC();
576: }
577: #endif
1.1 matt 578: TLBSYNC();
579: SYNC();
580: }
581:
1.35 perry 582: static inline register_t
1.2 matt 583: va_to_vsid(const struct pmap *pm, vaddr_t addr)
1.1 matt 584: {
1.18 matt 585: /*
1.102 thorpej 586: * Rather than searching the STE groups for the VSID or extracting
587: * it from the SR, we know how we generate that from the ESID and
588: * so do that.
589: *
590: * This makes the code the same for OEA and OEA64, and also allows
591: * us to generate a correct-for-that-address-space VSID even if the
592: * pmap contains a different SR value at any given moment (e.g.
593: * kernel pmap on a 601 that is using I/O segments).
1.18 matt 594: */
595: return VSID_MAKE(addr >> ADDR_SR_SHFT, pm->pm_vsid) >> SR_VSID_SHFT;
1.1 matt 596: }
597:
1.35 perry 598: static inline register_t
1.2 matt 599: va_to_pteg(const struct pmap *pm, vaddr_t addr)
1.1 matt 600: {
1.2 matt 601: register_t hash;
602:
603: hash = va_to_vsid(pm, addr) ^ ((addr & ADDR_PIDX) >> ADDR_PIDX_SHFT);
1.1 matt 604: return hash & pmap_pteg_mask;
605: }
606:
607: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
608: /*
609: * Given a PTE in the page table, calculate the VADDR that hashes to it.
610: * The only bit of magic is that the top 4 bits of the address doesn't
611: * technically exist in the PTE. But we know we reserved 4 bits of the
612: * VSID for it so that's how we get it.
613: */
614: static vaddr_t
1.2 matt 615: pmap_pte_to_va(volatile const struct pte *pt)
1.1 matt 616: {
617: vaddr_t va;
618: uintptr_t ptaddr = (uintptr_t) pt;
619:
620: if (pt->pte_hi & PTE_HID)
1.2 matt 621: ptaddr ^= (pmap_pteg_mask * sizeof(struct pteg));
1.1 matt 622:
1.18 matt 623: /* PPC Bits 10-19 PPC64 Bits 42-51 */
1.53 garbled 624: #if defined(PMAP_OEA)
1.4 matt 625: va = ((pt->pte_hi >> PTE_VSID_SHFT) ^ (ptaddr / sizeof(struct pteg))) & 0x3ff;
1.53 garbled 626: #elif defined (PMAP_OEA64) || defined (PMAP_OEA64_BRIDGE)
1.38 sanjayl 627: va = ((pt->pte_hi >> PTE_VSID_SHFT) ^ (ptaddr / sizeof(struct pteg))) & 0x7ff;
628: #endif
1.1 matt 629: va <<= ADDR_PIDX_SHFT;
630:
1.18 matt 631: /* PPC Bits 4-9 PPC64 Bits 36-41 */
1.1 matt 632: va |= (pt->pte_hi & PTE_API) << ADDR_API_SHFT;
633:
1.53 garbled 634: #if defined(PMAP_OEA64)
1.18 matt 635: /* PPC63 Bits 0-35 */
636: /* va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT; */
1.53 garbled 637: #elif defined(PMAP_OEA) || defined(PMAP_OEA64_BRIDGE)
1.1 matt 638: /* PPC Bits 0-3 */
639: va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT;
1.18 matt 640: #endif
1.1 matt 641:
642: return va;
643: }
644: #endif
645:
1.35 perry 646: static inline struct pvo_head *
1.1 matt 647: pa_to_pvoh(paddr_t pa, struct vm_page **pg_p)
648: {
649: struct vm_page *pg;
1.72 uebayasi 650: struct vm_page_md *md;
1.108 riastrad 651: struct pmap_page *pp;
1.1 matt 652:
653: pg = PHYS_TO_VM_PAGE(pa);
654: if (pg_p != NULL)
655: *pg_p = pg;
1.108 riastrad 656: if (pg == NULL) {
657: if ((pp = pmap_pv_tracked(pa)) != NULL)
658: return &pp->pp_pvoh;
1.107 chs 659: return NULL;
1.108 riastrad 660: }
1.72 uebayasi 661: md = VM_PAGE_TO_MD(pg);
662: return &md->mdpg_pvoh;
1.1 matt 663: }
664:
1.35 perry 665: static inline struct pvo_head *
1.1 matt 666: vm_page_to_pvoh(struct vm_page *pg)
667: {
1.72 uebayasi 668: struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
669:
670: return &md->mdpg_pvoh;
1.1 matt 671: }
672:
1.108 riastrad 673: static inline void
674: pmap_pp_attr_clear(struct pmap_page *pp, int ptebit)
675: {
676:
677: pp->pp_attrs &= ptebit;
678: }
1.1 matt 679:
1.35 perry 680: static inline void
1.1 matt 681: pmap_attr_clear(struct vm_page *pg, int ptebit)
682: {
1.72 uebayasi 683: struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
684:
1.108 riastrad 685: pmap_pp_attr_clear(&md->mdpg_pp, ptebit);
686: }
687:
688: static inline int
689: pmap_pp_attr_fetch(struct pmap_page *pp)
690: {
691:
692: return pp->pp_attrs;
1.1 matt 693: }
694:
1.35 perry 695: static inline int
1.1 matt 696: pmap_attr_fetch(struct vm_page *pg)
697: {
1.72 uebayasi 698: struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
699:
1.108 riastrad 700: return pmap_pp_attr_fetch(&md->mdpg_pp);
1.1 matt 701: }
702:
1.35 perry 703: static inline void
1.1 matt 704: pmap_attr_save(struct vm_page *pg, int ptebit)
705: {
1.72 uebayasi 706: struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
707:
708: md->mdpg_attrs |= ptebit;
1.1 matt 709: }
710:
1.35 perry 711: static inline int
1.2 matt 712: pmap_pte_compare(const volatile struct pte *pt, const struct pte *pvo_pt)
1.1 matt 713: {
714: if (pt->pte_hi == pvo_pt->pte_hi
715: #if 0
716: && ((pt->pte_lo ^ pvo_pt->pte_lo) &
717: ~(PTE_REF|PTE_CHG)) == 0
718: #endif
719: )
720: return 1;
721: return 0;
722: }
723:
1.35 perry 724: static inline void
1.2 matt 725: pmap_pte_create(struct pte *pt, const struct pmap *pm, vaddr_t va, register_t pte_lo)
1.1 matt 726: {
727: /*
728: * Construct the PTE. Default to IMB initially. Valid bit
729: * only gets set when the real pte is set in memory.
730: *
731: * Note: Don't set the valid bit for correct operation of tlb update.
732: */
1.53 garbled 733: #if defined(PMAP_OEA)
1.2 matt 734: pt->pte_hi = (va_to_vsid(pm, va) << PTE_VSID_SHFT)
735: | (((va & ADDR_PIDX) >> (ADDR_API_SHFT - PTE_API_SHFT)) & PTE_API);
1.1 matt 736: pt->pte_lo = pte_lo;
1.79 matt 737: #elif defined (PMAP_OEA64_BRIDGE) || defined (PMAP_OEA64)
1.38 sanjayl 738: pt->pte_hi = ((u_int64_t)va_to_vsid(pm, va) << PTE_VSID_SHFT)
739: | (((va & ADDR_PIDX) >> (ADDR_API_SHFT - PTE_API_SHFT)) & PTE_API);
740: pt->pte_lo = (u_int64_t) pte_lo;
1.53 garbled 741: #endif /* PMAP_OEA */
1.1 matt 742: }
743:
1.35 perry 744: static inline void
1.2 matt 745: pmap_pte_synch(volatile struct pte *pt, struct pte *pvo_pt)
1.1 matt 746: {
747: pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF|PTE_CHG);
748: }
749:
1.35 perry 750: static inline void
1.2 matt 751: pmap_pte_clear(volatile struct pte *pt, vaddr_t va, int ptebit)
1.1 matt 752: {
753: /*
754: * As shown in Section 7.6.3.2.3
755: */
756: pt->pte_lo &= ~ptebit;
757: TLBIE(va);
758: SYNC();
759: EIEIO();
760: TLBSYNC();
761: SYNC();
1.57 matt 762: #ifdef MULTIPROCESSOR
763: DCBST(pt);
764: #endif
1.1 matt 765: }
766:
1.35 perry 767: static inline void
1.2 matt 768: pmap_pte_set(volatile struct pte *pt, struct pte *pvo_pt)
1.1 matt 769: {
770: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
771: if (pvo_pt->pte_hi & PTE_VALID)
772: panic("pte_set: setting an already valid pte %p", pvo_pt);
773: #endif
774: pvo_pt->pte_hi |= PTE_VALID;
1.38 sanjayl 775:
1.1 matt 776: /*
777: * Update the PTE as defined in section 7.6.3.1
778: * Note that the REF/CHG bits are from pvo_pt and thus should
779: * have been saved so this routine can restore them (if desired).
780: */
781: pt->pte_lo = pvo_pt->pte_lo;
782: EIEIO();
783: pt->pte_hi = pvo_pt->pte_hi;
1.38 sanjayl 784: TLBSYNC();
1.1 matt 785: SYNC();
1.57 matt 786: #ifdef MULTIPROCESSOR
787: DCBST(pt);
788: #endif
1.1 matt 789: pmap_pte_valid++;
790: }
791:
1.35 perry 792: static inline void
1.2 matt 793: pmap_pte_unset(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va)
1.1 matt 794: {
795: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
796: if ((pvo_pt->pte_hi & PTE_VALID) == 0)
797: panic("pte_unset: attempt to unset an inactive pte#1 %p/%p", pvo_pt, pt);
798: if ((pt->pte_hi & PTE_VALID) == 0)
799: panic("pte_unset: attempt to unset an inactive pte#2 %p/%p", pvo_pt, pt);
800: #endif
801:
802: pvo_pt->pte_hi &= ~PTE_VALID;
803: /*
804: * Force the ref & chg bits back into the PTEs.
805: */
806: SYNC();
807: /*
808: * Invalidate the pte ... (Section 7.6.3.3)
809: */
810: pt->pte_hi &= ~PTE_VALID;
811: SYNC();
812: TLBIE(va);
813: SYNC();
814: EIEIO();
815: TLBSYNC();
816: SYNC();
817: /*
818: * Save the ref & chg bits ...
819: */
820: pmap_pte_synch(pt, pvo_pt);
821: pmap_pte_valid--;
822: }
823:
1.35 perry 824: static inline void
1.2 matt 825: pmap_pte_change(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va)
1.1 matt 826: {
827: /*
828: * Invalidate the PTE
829: */
830: pmap_pte_unset(pt, pvo_pt, va);
831: pmap_pte_set(pt, pvo_pt);
832: }
833:
834: /*
835: * Try to insert the PTE @ *pvo_pt into the pmap_pteg_table at ptegidx
836: * (either primary or secondary location).
837: *
838: * Note: both the destination and source PTEs must not have PTE_VALID set.
839: */
840:
1.53 garbled 841: static int
1.2 matt 842: pmap_pte_insert(int ptegidx, struct pte *pvo_pt)
1.1 matt 843: {
1.2 matt 844: volatile struct pte *pt;
1.1 matt 845: int i;
846:
847: #if defined(DEBUG)
1.85 matt 848: DPRINTFN(PTE, "pmap_pte_insert: idx %#x, pte %#" _PRIxpte " %#" _PRIxpte "\n",
849: ptegidx, pvo_pt->pte_hi, pvo_pt->pte_lo);
1.1 matt 850: #endif
851: /*
852: * First try primary hash.
853: */
854: for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
855: if ((pt->pte_hi & PTE_VALID) == 0) {
856: pvo_pt->pte_hi &= ~PTE_HID;
857: pmap_pte_set(pt, pvo_pt);
858: return i;
859: }
860: }
861:
862: /*
863: * Now try secondary hash.
864: */
865: ptegidx ^= pmap_pteg_mask;
866: for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
867: if ((pt->pte_hi & PTE_VALID) == 0) {
868: pvo_pt->pte_hi |= PTE_HID;
869: pmap_pte_set(pt, pvo_pt);
870: return i;
871: }
872: }
873: return -1;
874: }
875:
876: /*
877: * Spill handler.
878: *
879: * Tries to spill a page table entry from the overflow area.
880: * This runs in either real mode (if dealing with a exception spill)
881: * or virtual mode when dealing with manually spilling one of the
882: * kernel's pte entries. In either case, interrupts are already
883: * disabled.
884: */
1.14 chs 885:
1.1 matt 886: int
1.44 thorpej 887: pmap_pte_spill(struct pmap *pm, vaddr_t addr, bool exec)
1.1 matt 888: {
889: struct pvo_entry *source_pvo, *victim_pvo, *next_pvo;
890: struct pvo_entry *pvo;
1.15 dyoung 891: /* XXX: gcc -- vpvoh is always set at either *1* or *2* */
892: struct pvo_tqhead *pvoh, *vpvoh = NULL;
1.1 matt 893: int ptegidx, i, j;
1.2 matt 894: volatile struct pteg *pteg;
895: volatile struct pte *pt;
1.1 matt 896:
1.50 ad 897: PMAP_LOCK();
898:
1.2 matt 899: ptegidx = va_to_pteg(pm, addr);
1.1 matt 900:
901: /*
902: * Have to substitute some entry. Use the primary hash for this.
1.12 matt 903: * Use low bits of timebase as random generator. Make sure we are
904: * not picking a kernel pte for replacement.
1.1 matt 905: */
906: pteg = &pmap_pteg_table[ptegidx];
907: i = MFTB() & 7;
1.12 matt 908: for (j = 0; j < 8; j++) {
909: pt = &pteg->pt[i];
1.53 garbled 910: if ((pt->pte_hi & PTE_VALID) == 0)
911: break;
912: if (VSID_TO_HASH((pt->pte_hi & PTE_VSID) >> PTE_VSID_SHFT)
913: < PHYSMAP_VSIDBITS)
1.12 matt 914: break;
915: i = (i + 1) & 7;
916: }
917: KASSERT(j < 8);
1.1 matt 918:
919: source_pvo = NULL;
920: victim_pvo = NULL;
921: pvoh = &pmap_pvo_table[ptegidx];
922: TAILQ_FOREACH(pvo, pvoh, pvo_olink) {
923:
924: /*
925: * We need to find pvo entry for this address...
926: */
927: PMAP_PVO_CHECK(pvo); /* sanity check */
928:
929: /*
930: * If we haven't found the source and we come to a PVO with
931: * a valid PTE, then we know we can't find it because all
932: * evicted PVOs always are first in the list.
933: */
934: if (source_pvo == NULL && (pvo->pvo_pte.pte_hi & PTE_VALID))
935: break;
1.2 matt 936: if (source_pvo == NULL && pm == pvo->pvo_pmap &&
937: addr == PVO_VADDR(pvo)) {
1.1 matt 938:
939: /*
940: * Now we have found the entry to be spilled into the
941: * pteg. Attempt to insert it into the page table.
942: */
943: j = pmap_pte_insert(ptegidx, &pvo->pvo_pte);
944: if (j >= 0) {
945: PVO_PTEGIDX_SET(pvo, j);
946: PMAP_PVO_CHECK(pvo); /* sanity check */
1.12 matt 947: PVO_WHERE(pvo, SPILL_INSERT);
1.1 matt 948: pvo->pvo_pmap->pm_evictions--;
949: PMAPCOUNT(ptes_spilled);
950: PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID)
951: ? pmap_evcnt_ptes_secondary
952: : pmap_evcnt_ptes_primary)[j]);
953:
954: /*
955: * Since we keep the evicted entries at the
956: * from of the PVO list, we need move this
957: * (now resident) PVO after the evicted
958: * entries.
959: */
960: next_pvo = TAILQ_NEXT(pvo, pvo_olink);
961:
962: /*
1.5 matt 963: * If we don't have to move (either we were the
964: * last entry or the next entry was valid),
1.1 matt 965: * don't change our position. Otherwise
966: * move ourselves to the tail of the queue.
967: */
968: if (next_pvo != NULL &&
969: !(next_pvo->pvo_pte.pte_hi & PTE_VALID)) {
970: TAILQ_REMOVE(pvoh, pvo, pvo_olink);
971: TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink);
972: }
1.50 ad 973: PMAP_UNLOCK();
1.1 matt 974: return 1;
975: }
976: source_pvo = pvo;
1.39 matt 977: if (exec && !PVO_EXECUTABLE_P(source_pvo)) {
1.96 rin 978: PMAP_UNLOCK();
1.14 chs 979: return 0;
980: }
1.1 matt 981: if (victim_pvo != NULL)
982: break;
983: }
984:
985: /*
986: * We also need the pvo entry of the victim we are replacing
987: * so save the R & C bits of the PTE.
988: */
989: if ((pt->pte_hi & PTE_HID) == 0 && victim_pvo == NULL &&
990: pmap_pte_compare(pt, &pvo->pvo_pte)) {
1.15 dyoung 991: vpvoh = pvoh; /* *1* */
1.1 matt 992: victim_pvo = pvo;
993: if (source_pvo != NULL)
994: break;
995: }
996: }
997:
998: if (source_pvo == NULL) {
999: PMAPCOUNT(ptes_unspilled);
1.50 ad 1000: PMAP_UNLOCK();
1.1 matt 1001: return 0;
1002: }
1003:
1004: if (victim_pvo == NULL) {
1005: if ((pt->pte_hi & PTE_HID) == 0)
1006: panic("pmap_pte_spill: victim p-pte (%p) has "
1007: "no pvo entry!", pt);
1008:
1009: /*
1010: * If this is a secondary PTE, we need to search
1011: * its primary pvo bucket for the matching PVO.
1012: */
1.15 dyoung 1013: vpvoh = &pmap_pvo_table[ptegidx ^ pmap_pteg_mask]; /* *2* */
1.1 matt 1014: TAILQ_FOREACH(pvo, vpvoh, pvo_olink) {
1015: PMAP_PVO_CHECK(pvo); /* sanity check */
1016:
1017: /*
1018: * We also need the pvo entry of the victim we are
1019: * replacing so save the R & C bits of the PTE.
1020: */
1021: if (pmap_pte_compare(pt, &pvo->pvo_pte)) {
1022: victim_pvo = pvo;
1023: break;
1024: }
1025: }
1026: if (victim_pvo == NULL)
1027: panic("pmap_pte_spill: victim s-pte (%p) has "
1028: "no pvo entry!", pt);
1029: }
1030:
1031: /*
1.12 matt 1032: * The victim should be not be a kernel PVO/PTE entry.
1033: */
1034: KASSERT(victim_pvo->pvo_pmap != pmap_kernel());
1035: KASSERT(PVO_PTEGIDX_ISSET(victim_pvo));
1036: KASSERT(PVO_PTEGIDX_GET(victim_pvo) == i);
1037:
1038: /*
1.1 matt 1039: * We are invalidating the TLB entry for the EA for the
1040: * we are replacing even though its valid; If we don't
1041: * we lose any ref/chg bit changes contained in the TLB
1042: * entry.
1043: */
1044: source_pvo->pvo_pte.pte_hi &= ~PTE_HID;
1045:
1046: /*
1047: * To enforce the PVO list ordering constraint that all
1048: * evicted entries should come before all valid entries,
1049: * move the source PVO to the tail of its list and the
1050: * victim PVO to the head of its list (which might not be
1051: * the same list, if the victim was using the secondary hash).
1052: */
1053: TAILQ_REMOVE(pvoh, source_pvo, pvo_olink);
1054: TAILQ_INSERT_TAIL(pvoh, source_pvo, pvo_olink);
1055: TAILQ_REMOVE(vpvoh, victim_pvo, pvo_olink);
1056: TAILQ_INSERT_HEAD(vpvoh, victim_pvo, pvo_olink);
1057: pmap_pte_unset(pt, &victim_pvo->pvo_pte, victim_pvo->pvo_vaddr);
1058: pmap_pte_set(pt, &source_pvo->pvo_pte);
1059: victim_pvo->pvo_pmap->pm_evictions++;
1060: source_pvo->pvo_pmap->pm_evictions--;
1.12 matt 1061: PVO_WHERE(victim_pvo, SPILL_UNSET);
1062: PVO_WHERE(source_pvo, SPILL_SET);
1.1 matt 1063:
1064: PVO_PTEGIDX_CLR(victim_pvo);
1065: PVO_PTEGIDX_SET(source_pvo, i);
1066: PMAPCOUNT2(pmap_evcnt_ptes_primary[i]);
1067: PMAPCOUNT(ptes_spilled);
1068: PMAPCOUNT(ptes_evicted);
1069: PMAPCOUNT(ptes_removed);
1070:
1071: PMAP_PVO_CHECK(victim_pvo);
1072: PMAP_PVO_CHECK(source_pvo);
1.50 ad 1073:
1074: PMAP_UNLOCK();
1.1 matt 1075: return 1;
1076: }
1077:
1078: /*
1079: * Restrict given range to physical memory
1080: */
1081: void
1082: pmap_real_memory(paddr_t *start, psize_t *size)
1083: {
1084: struct mem_region *mp;
1085:
1086: for (mp = mem; mp->size; mp++) {
1087: if (*start + *size > mp->start
1088: && *start < mp->start + mp->size) {
1089: if (*start < mp->start) {
1090: *size -= mp->start - *start;
1091: *start = mp->start;
1092: }
1093: if (*start + *size > mp->start + mp->size)
1094: *size = mp->start + mp->size - *start;
1095: return;
1096: }
1097: }
1098: *size = 0;
1099: }
1100:
1101: /*
1102: * Initialize anything else for pmap handling.
1103: * Called during vm_init().
1104: */
1105: void
1106: pmap_init(void)
1107: {
1108:
1109: pmap_initialized = 1;
1110: }
1111:
1112: /*
1.10 thorpej 1113: * How much virtual space does the kernel get?
1114: */
1115: void
1116: pmap_virtual_space(vaddr_t *start, vaddr_t *end)
1117: {
1118: /*
1119: * For now, reserve one segment (minus some overhead) for kernel
1120: * virtual memory
1121: */
1122: *start = VM_MIN_KERNEL_ADDRESS;
1123: *end = VM_MAX_KERNEL_ADDRESS;
1124: }
1125:
1126: /*
1.1 matt 1127: * Allocate, initialize, and return a new physical map.
1128: */
1129: pmap_t
1130: pmap_create(void)
1131: {
1132: pmap_t pm;
1.38 sanjayl 1133:
1.1 matt 1134: pm = pool_get(&pmap_pool, PR_WAITOK);
1.84 matt 1135: KASSERT((vaddr_t)pm < VM_MIN_KERNEL_ADDRESS);
1.46 christos 1136: memset((void *)pm, 0, sizeof *pm);
1.1 matt 1137: pmap_pinit(pm);
1138:
1.85 matt 1139: DPRINTFN(CREATE, "pmap_create: pm %p:\n"
1.54 mlelstv 1140: "\t%#" _PRIsr " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr
1141: " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr "\n"
1142: "\t%#" _PRIsr " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr
1143: " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr "\n",
1144: pm,
1145: pm->pm_sr[0], pm->pm_sr[1],
1146: pm->pm_sr[2], pm->pm_sr[3],
1147: pm->pm_sr[4], pm->pm_sr[5],
1148: pm->pm_sr[6], pm->pm_sr[7],
1149: pm->pm_sr[8], pm->pm_sr[9],
1150: pm->pm_sr[10], pm->pm_sr[11],
1151: pm->pm_sr[12], pm->pm_sr[13],
1.85 matt 1152: pm->pm_sr[14], pm->pm_sr[15]);
1.1 matt 1153: return pm;
1154: }
1155:
1156: /*
1157: * Initialize a preallocated and zeroed pmap structure.
1158: */
1159: void
1160: pmap_pinit(pmap_t pm)
1161: {
1.2 matt 1162: register_t entropy = MFTB();
1163: register_t mask;
1164: int i;
1.1 matt 1165:
1166: /*
1167: * Allocate some segment registers for this pmap.
1168: */
1169: pm->pm_refs = 1;
1.50 ad 1170: PMAP_LOCK();
1.2 matt 1171: for (i = 0; i < NPMAPS; i += VSID_NBPW) {
1172: static register_t pmap_vsidcontext;
1173: register_t hash;
1174: unsigned int n;
1.1 matt 1175:
1176: /* Create a new value by multiplying by a prime adding in
1177: * entropy from the timebase register. This is to make the
1178: * VSID more random so that the PT Hash function collides
1179: * less often. (note that the prime causes gcc to do shifts
1180: * instead of a multiply)
1181: */
1182: pmap_vsidcontext = (pmap_vsidcontext * 0x1105) + entropy;
1183: hash = pmap_vsidcontext & (NPMAPS - 1);
1.23 aymeric 1184: if (hash == 0) { /* 0 is special, avoid it */
1185: entropy += 0xbadf00d;
1.1 matt 1186: continue;
1.23 aymeric 1187: }
1.1 matt 1188: n = hash >> 5;
1.2 matt 1189: mask = 1L << (hash & (VSID_NBPW-1));
1190: hash = pmap_vsidcontext;
1.1 matt 1191: if (pmap_vsid_bitmap[n] & mask) { /* collision? */
1192: /* anything free in this bucket? */
1.2 matt 1193: if (~pmap_vsid_bitmap[n] == 0) {
1.23 aymeric 1194: entropy = hash ^ (hash >> 16);
1.1 matt 1195: continue;
1196: }
1197: i = ffs(~pmap_vsid_bitmap[n]) - 1;
1.2 matt 1198: mask = 1L << i;
1199: hash &= ~(VSID_NBPW-1);
1.1 matt 1200: hash |= i;
1201: }
1.18 matt 1202: hash &= PTE_VSID >> PTE_VSID_SHFT;
1.1 matt 1203: pmap_vsid_bitmap[n] |= mask;
1.18 matt 1204: pm->pm_vsid = hash;
1.53 garbled 1205: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1 matt 1206: for (i = 0; i < 16; i++)
1.14 chs 1207: pm->pm_sr[i] = VSID_MAKE(i, hash) | SR_PRKEY |
1208: SR_NOEXEC;
1.18 matt 1209: #endif
1.50 ad 1210: PMAP_UNLOCK();
1.1 matt 1211: return;
1212: }
1.50 ad 1213: PMAP_UNLOCK();
1.1 matt 1214: panic("pmap_pinit: out of segments");
1215: }
1216:
1217: /*
1218: * Add a reference to the given pmap.
1219: */
1220: void
1221: pmap_reference(pmap_t pm)
1222: {
1.50 ad 1223: atomic_inc_uint(&pm->pm_refs);
1.1 matt 1224: }
1225:
1226: /*
1227: * Retire the given pmap from service.
1228: * Should only be called if the map contains no valid mappings.
1229: */
1230: void
1231: pmap_destroy(pmap_t pm)
1232: {
1.50 ad 1233: if (atomic_dec_uint_nv(&pm->pm_refs) == 0) {
1.1 matt 1234: pmap_release(pm);
1235: pool_put(&pmap_pool, pm);
1236: }
1237: }
1238:
1239: /*
1240: * Release any resources held by the given physical map.
1241: * Called when a pmap initialized by pmap_pinit is being released.
1242: */
1243: void
1244: pmap_release(pmap_t pm)
1245: {
1246: int idx, mask;
1.39 matt 1247:
1248: KASSERT(pm->pm_stats.resident_count == 0);
1249: KASSERT(pm->pm_stats.wired_count == 0);
1.1 matt 1250:
1.50 ad 1251: PMAP_LOCK();
1.1 matt 1252: if (pm->pm_sr[0] == 0)
1253: panic("pmap_release");
1.22 aymeric 1254: idx = pm->pm_vsid & (NPMAPS-1);
1.1 matt 1255: mask = 1 << (idx % VSID_NBPW);
1256: idx /= VSID_NBPW;
1.22 aymeric 1257:
1258: KASSERT(pmap_vsid_bitmap[idx] & mask);
1.1 matt 1259: pmap_vsid_bitmap[idx] &= ~mask;
1.50 ad 1260: PMAP_UNLOCK();
1.1 matt 1261: }
1262:
1263: /*
1264: * Copy the range specified by src_addr/len
1265: * from the source map to the range dst_addr/len
1266: * in the destination map.
1267: *
1268: * This routine is only advisory and need not do anything.
1269: */
1270: void
1271: pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr,
1272: vsize_t len, vaddr_t src_addr)
1273: {
1274: PMAPCOUNT(copies);
1275: }
1276:
1277: /*
1278: * Require that all active physical maps contain no
1279: * incorrect entries NOW.
1280: */
1281: void
1282: pmap_update(struct pmap *pmap)
1283: {
1284: PMAPCOUNT(updates);
1285: TLBSYNC();
1286: }
1287:
1.35 perry 1288: static inline int
1.1 matt 1289: pmap_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx)
1290: {
1291: int pteidx;
1292: /*
1293: * We can find the actual pte entry without searching by
1294: * grabbing the PTEG index from 3 unused bits in pte_lo[11:9]
1295: * and by noticing the HID bit.
1296: */
1297: pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo);
1298: if (pvo->pvo_pte.pte_hi & PTE_HID)
1299: pteidx ^= pmap_pteg_mask * 8;
1300: return pteidx;
1301: }
1302:
1.2 matt 1303: volatile struct pte *
1.1 matt 1304: pmap_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
1305: {
1.2 matt 1306: volatile struct pte *pt;
1.1 matt 1307:
1308: #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK)
1309: if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0)
1310: return NULL;
1311: #endif
1312:
1313: /*
1314: * If we haven't been supplied the ptegidx, calculate it.
1315: */
1316: if (pteidx == -1) {
1317: int ptegidx;
1.2 matt 1318: ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr);
1.1 matt 1319: pteidx = pmap_pvo_pte_index(pvo, ptegidx);
1320: }
1321:
1322: pt = &pmap_pteg_table[pteidx >> 3].pt[pteidx & 7];
1323:
1324: #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK)
1325: return pt;
1326: #else
1327: if ((pvo->pvo_pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) {
1328: panic("pmap_pvo_to_pte: pvo %p: has valid pte in "
1329: "pvo but no valid pte index", pvo);
1330: }
1331: if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) {
1332: panic("pmap_pvo_to_pte: pvo %p: has valid pte index in "
1333: "pvo but no valid pte", pvo);
1334: }
1335:
1336: if ((pt->pte_hi ^ (pvo->pvo_pte.pte_hi & ~PTE_VALID)) == PTE_VALID) {
1337: if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) {
1338: #if defined(DEBUG) || defined(PMAPCHECK)
1339: pmap_pte_print(pt);
1340: #endif
1341: panic("pmap_pvo_to_pte: pvo %p: has valid pte in "
1342: "pmap_pteg_table %p but invalid in pvo",
1343: pvo, pt);
1344: }
1345: if (((pt->pte_lo ^ pvo->pvo_pte.pte_lo) & ~(PTE_CHG|PTE_REF)) != 0) {
1346: #if defined(DEBUG) || defined(PMAPCHECK)
1347: pmap_pte_print(pt);
1348: #endif
1349: panic("pmap_pvo_to_pte: pvo %p: pvo pte does "
1350: "not match pte %p in pmap_pteg_table",
1351: pvo, pt);
1352: }
1353: return pt;
1354: }
1355:
1356: if (pvo->pvo_pte.pte_hi & PTE_VALID) {
1357: #if defined(DEBUG) || defined(PMAPCHECK)
1358: pmap_pte_print(pt);
1359: #endif
1.12 matt 1360: panic("pmap_pvo_to_pte: pvo %p: has nomatching pte %p in "
1.1 matt 1361: "pmap_pteg_table but valid in pvo", pvo, pt);
1362: }
1363: return NULL;
1364: #endif /* !(!DIAGNOSTIC && !DEBUG && !PMAPCHECK) */
1365: }
1366:
1367: struct pvo_entry *
1368: pmap_pvo_find_va(pmap_t pm, vaddr_t va, int *pteidx_p)
1369: {
1370: struct pvo_entry *pvo;
1371: int ptegidx;
1372:
1373: va &= ~ADDR_POFF;
1.2 matt 1374: ptegidx = va_to_pteg(pm, va);
1.1 matt 1375:
1376: TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
1377: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1378: if ((uintptr_t) pvo >= SEGMENT_LENGTH)
1379: panic("pmap_pvo_find_va: invalid pvo %p on "
1380: "list %#x (%p)", pvo, ptegidx,
1381: &pmap_pvo_table[ptegidx]);
1382: #endif
1383: if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
1384: if (pteidx_p)
1385: *pteidx_p = pmap_pvo_pte_index(pvo, ptegidx);
1386: return pvo;
1387: }
1388: }
1.38 sanjayl 1389: if ((pm == pmap_kernel()) && (va < SEGMENT_LENGTH))
1.54 mlelstv 1390: panic("%s: returning NULL for %s pmap, va: %#" _PRIxva "\n",
1.53 garbled 1391: __func__, (pm == pmap_kernel() ? "kernel" : "user"), va);
1.1 matt 1392: return NULL;
1393: }
1394:
1395: #if defined(DEBUG) || defined(PMAPCHECK)
1396: void
1397: pmap_pvo_check(const struct pvo_entry *pvo)
1398: {
1399: struct pvo_head *pvo_head;
1400: struct pvo_entry *pvo0;
1.2 matt 1401: volatile struct pte *pt;
1.1 matt 1402: int failed = 0;
1403:
1.50 ad 1404: PMAP_LOCK();
1405:
1.1 matt 1406: if ((uintptr_t)(pvo+1) >= SEGMENT_LENGTH)
1407: panic("pmap_pvo_check: pvo %p: invalid address", pvo);
1408:
1409: if ((uintptr_t)(pvo->pvo_pmap+1) >= SEGMENT_LENGTH) {
1410: printf("pmap_pvo_check: pvo %p: invalid pmap address %p\n",
1411: pvo, pvo->pvo_pmap);
1412: failed = 1;
1413: }
1414:
1415: if ((uintptr_t)TAILQ_NEXT(pvo, pvo_olink) >= SEGMENT_LENGTH ||
1416: (((uintptr_t)TAILQ_NEXT(pvo, pvo_olink)) & 0x1f) != 0) {
1417: printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n",
1418: pvo, TAILQ_NEXT(pvo, pvo_olink));
1419: failed = 1;
1420: }
1421:
1422: if ((uintptr_t)LIST_NEXT(pvo, pvo_vlink) >= SEGMENT_LENGTH ||
1423: (((uintptr_t)LIST_NEXT(pvo, pvo_vlink)) & 0x1f) != 0) {
1424: printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n",
1425: pvo, LIST_NEXT(pvo, pvo_vlink));
1426: failed = 1;
1427: }
1428:
1.39 matt 1429: if (PVO_MANAGED_P(pvo)) {
1.1 matt 1430: pvo_head = pa_to_pvoh(pvo->pvo_pte.pte_lo & PTE_RPGN, NULL);
1.107 chs 1431: LIST_FOREACH(pvo0, pvo_head, pvo_vlink) {
1432: if (pvo0 == pvo)
1433: break;
1434: }
1435: if (pvo0 == NULL) {
1436: printf("pmap_pvo_check: pvo %p: not present "
1437: "on its vlist head %p\n", pvo, pvo_head);
1438: failed = 1;
1439: }
1.1 matt 1440: } else {
1.107 chs 1441: KASSERT(pvo->pvo_vaddr >= VM_MIN_KERNEL_ADDRESS);
1442: if (__predict_false(pvo->pvo_vaddr < VM_MIN_KERNEL_ADDRESS))
1.1 matt 1443: failed = 1;
1444: }
1445: if (pvo != pmap_pvo_find_va(pvo->pvo_pmap, pvo->pvo_vaddr, NULL)) {
1446: printf("pmap_pvo_check: pvo %p: not present "
1447: "on its olist head\n", pvo);
1448: failed = 1;
1449: }
1450: pt = pmap_pvo_to_pte(pvo, -1);
1451: if (pt == NULL) {
1452: if (pvo->pvo_pte.pte_hi & PTE_VALID) {
1453: printf("pmap_pvo_check: pvo %p: pte_hi VALID but "
1454: "no PTE\n", pvo);
1455: failed = 1;
1456: }
1457: } else {
1458: if ((uintptr_t) pt < (uintptr_t) &pmap_pteg_table[0] ||
1459: (uintptr_t) pt >=
1460: (uintptr_t) &pmap_pteg_table[pmap_pteg_cnt]) {
1461: printf("pmap_pvo_check: pvo %p: pte %p not in "
1462: "pteg table\n", pvo, pt);
1463: failed = 1;
1464: }
1465: if (((((uintptr_t) pt) >> 3) & 7) != PVO_PTEGIDX_GET(pvo)) {
1466: printf("pmap_pvo_check: pvo %p: pte_hi VALID but "
1467: "no PTE\n", pvo);
1468: failed = 1;
1469: }
1470: if (pvo->pvo_pte.pte_hi != pt->pte_hi) {
1471: printf("pmap_pvo_check: pvo %p: pte_hi differ: "
1.54 mlelstv 1472: "%#" _PRIxpte "/%#" _PRIxpte "\n", pvo,
1473: pvo->pvo_pte.pte_hi,
1474: pt->pte_hi);
1.1 matt 1475: failed = 1;
1476: }
1477: if (((pvo->pvo_pte.pte_lo ^ pt->pte_lo) &
1478: (PTE_PP|PTE_WIMG|PTE_RPGN)) != 0) {
1479: printf("pmap_pvo_check: pvo %p: pte_lo differ: "
1.54 mlelstv 1480: "%#" _PRIxpte "/%#" _PRIxpte "\n", pvo,
1481: (pvo->pvo_pte.pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN)),
1482: (pt->pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN)));
1.1 matt 1483: failed = 1;
1484: }
1485: if ((pmap_pte_to_va(pt) ^ PVO_VADDR(pvo)) & 0x0fffffff) {
1.53 garbled 1486: printf("pmap_pvo_check: pvo %p: PTE %p derived VA %#" _PRIxva ""
1487: " doesn't not match PVO's VA %#" _PRIxva "\n",
1.1 matt 1488: pvo, pt, pmap_pte_to_va(pt), PVO_VADDR(pvo));
1489: failed = 1;
1490: }
1491: if (failed)
1492: pmap_pte_print(pt);
1493: }
1494: if (failed)
1495: panic("pmap_pvo_check: pvo %p, pm %p: bugcheck!", pvo,
1496: pvo->pvo_pmap);
1.50 ad 1497:
1498: PMAP_UNLOCK();
1.1 matt 1499: }
1500: #endif /* DEBUG || PMAPCHECK */
1501:
1502: /*
1.25 chs 1503: * Search the PVO table looking for a non-wired entry.
1504: * If we find one, remove it and return it.
1505: */
1506:
1507: struct pvo_entry *
1508: pmap_pvo_reclaim(struct pmap *pm)
1509: {
1510: struct pvo_tqhead *pvoh;
1511: struct pvo_entry *pvo;
1512: uint32_t idx, endidx;
1513:
1514: endidx = pmap_pvo_reclaim_nextidx;
1515: for (idx = (endidx + 1) & pmap_pteg_mask; idx != endidx;
1516: idx = (idx + 1) & pmap_pteg_mask) {
1517: pvoh = &pmap_pvo_table[idx];
1518: TAILQ_FOREACH(pvo, pvoh, pvo_olink) {
1.39 matt 1519: if (!PVO_WIRED_P(pvo)) {
1.33 chs 1520: pmap_pvo_remove(pvo, -1, NULL);
1.25 chs 1521: pmap_pvo_reclaim_nextidx = idx;
1.26 matt 1522: PMAPCOUNT(pvos_reclaimed);
1.25 chs 1523: return pvo;
1524: }
1525: }
1526: }
1527: return NULL;
1528: }
1529:
1530: /*
1.1 matt 1531: * This returns whether this is the first mapping of a page.
1532: */
1533: int
1534: pmap_pvo_enter(pmap_t pm, struct pool *pl, struct pvo_head *pvo_head,
1.2 matt 1535: vaddr_t va, paddr_t pa, register_t pte_lo, int flags)
1.1 matt 1536: {
1537: struct pvo_entry *pvo;
1538: struct pvo_tqhead *pvoh;
1.2 matt 1539: register_t msr;
1.1 matt 1540: int ptegidx;
1541: int i;
1542: int poolflags = PR_NOWAIT;
1543:
1.28 chs 1544: /*
1545: * Compute the PTE Group index.
1546: */
1547: va &= ~ADDR_POFF;
1548: ptegidx = va_to_pteg(pm, va);
1549:
1550: msr = pmap_interrupts_off();
1551:
1.1 matt 1552: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1553: if (pmap_pvo_remove_depth > 0)
1554: panic("pmap_pvo_enter: called while pmap_pvo_remove active!");
1555: if (++pmap_pvo_enter_depth > 1)
1556: panic("pmap_pvo_enter: called recursively!");
1557: #endif
1558:
1559: /*
1560: * Remove any existing mapping for this page. Reuse the
1561: * pvo entry if there a mapping.
1562: */
1563: TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
1564: if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
1565: #ifdef DEBUG
1566: if ((pmapdebug & PMAPDEBUG_PVOENTER) &&
1567: ((pvo->pvo_pte.pte_lo ^ (pa|pte_lo)) &
1568: ~(PTE_REF|PTE_CHG)) == 0 &&
1569: va < VM_MIN_KERNEL_ADDRESS) {
1.56 phx 1570: printf("pmap_pvo_enter: pvo %p: dup %#" _PRIxpte "/%#" _PRIxpa "\n",
1.54 mlelstv 1571: pvo, pvo->pvo_pte.pte_lo, pte_lo|pa);
1.56 phx 1572: printf("pmap_pvo_enter: pte_hi=%#" _PRIxpte " sr=%#" _PRIsr "\n",
1.54 mlelstv 1573: pvo->pvo_pte.pte_hi,
1574: pm->pm_sr[va >> ADDR_SR_SHFT]);
1.1 matt 1575: pmap_pte_print(pmap_pvo_to_pte(pvo, -1));
1576: #ifdef DDBX
1577: Debugger();
1578: #endif
1579: }
1580: #endif
1581: PMAPCOUNT(mappings_replaced);
1.33 chs 1582: pmap_pvo_remove(pvo, -1, NULL);
1.1 matt 1583: break;
1584: }
1585: }
1586:
1587: /*
1588: * If we aren't overwriting an mapping, try to allocate
1589: */
1.26 matt 1590: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1591: --pmap_pvo_enter_depth;
1592: #endif
1.1 matt 1593: pmap_interrupts_restore(msr);
1.106 martin 1594: if (pvo == NULL) {
1.95 chs 1595: pvo = pool_get(pl, poolflags);
1.33 chs 1596: }
1.84 matt 1597: KASSERT((vaddr_t)pvo < VM_MIN_KERNEL_ADDRESS);
1.25 chs 1598:
1599: #ifdef DEBUG
1600: /*
1601: * Exercise pmap_pvo_reclaim() a little.
1602: */
1603: if (pvo && (flags & PMAP_CANFAIL) != 0 &&
1604: pmap_pvo_reclaim_debugctr++ > 0x1000 &&
1605: (pmap_pvo_reclaim_debugctr & 0xff) == 0) {
1606: pool_put(pl, pvo);
1607: pvo = NULL;
1608: }
1609: #endif
1610:
1.1 matt 1611: msr = pmap_interrupts_off();
1.26 matt 1612: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1613: ++pmap_pvo_enter_depth;
1614: #endif
1.1 matt 1615: if (pvo == NULL) {
1616: pvo = pmap_pvo_reclaim(pm);
1617: if (pvo == NULL) {
1618: if ((flags & PMAP_CANFAIL) == 0)
1619: panic("pmap_pvo_enter: failed");
1620: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1621: pmap_pvo_enter_depth--;
1622: #endif
1.26 matt 1623: PMAPCOUNT(pvos_failed);
1.1 matt 1624: pmap_interrupts_restore(msr);
1625: return ENOMEM;
1626: }
1627: }
1.25 chs 1628:
1.1 matt 1629: pvo->pvo_vaddr = va;
1630: pvo->pvo_pmap = pm;
1631: pvo->pvo_vaddr &= ~ADDR_POFF;
1632: if (flags & VM_PROT_EXECUTE) {
1633: PMAPCOUNT(exec_mappings);
1.14 chs 1634: pvo_set_exec(pvo);
1.1 matt 1635: }
1636: if (flags & PMAP_WIRED)
1637: pvo->pvo_vaddr |= PVO_WIRED;
1.107 chs 1638: if (pvo_head != NULL) {
1.1 matt 1639: pvo->pvo_vaddr |= PVO_MANAGED;
1640: PMAPCOUNT(mappings);
1641: } else {
1642: PMAPCOUNT(kernel_mappings);
1643: }
1.2 matt 1644: pmap_pte_create(&pvo->pvo_pte, pm, va, pa | pte_lo);
1.1 matt 1645:
1.107 chs 1646: if (pvo_head != NULL)
1647: LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink);
1.39 matt 1648: if (PVO_WIRED_P(pvo))
1.1 matt 1649: pvo->pvo_pmap->pm_stats.wired_count++;
1650: pvo->pvo_pmap->pm_stats.resident_count++;
1651: #if defined(DEBUG)
1.38 sanjayl 1652: /* if (pm != pmap_kernel() && va < VM_MIN_KERNEL_ADDRESS) */
1.1 matt 1653: DPRINTFN(PVOENTER,
1.85 matt 1654: "pmap_pvo_enter: pvo %p: pm %p va %#" _PRIxva " pa %#" _PRIxpa "\n",
1655: pvo, pm, va, pa);
1.1 matt 1656: #endif
1657:
1658: /*
1659: * We hope this succeeds but it isn't required.
1660: */
1661: pvoh = &pmap_pvo_table[ptegidx];
1662: i = pmap_pte_insert(ptegidx, &pvo->pvo_pte);
1663: if (i >= 0) {
1664: PVO_PTEGIDX_SET(pvo, i);
1.12 matt 1665: PVO_WHERE(pvo, ENTER_INSERT);
1.1 matt 1666: PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID)
1667: ? pmap_evcnt_ptes_secondary : pmap_evcnt_ptes_primary)[i]);
1668: TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink);
1.38 sanjayl 1669:
1.1 matt 1670: } else {
1671: /*
1672: * Since we didn't have room for this entry (which makes it
1673: * and evicted entry), place it at the head of the list.
1674: */
1675: TAILQ_INSERT_HEAD(pvoh, pvo, pvo_olink);
1676: PMAPCOUNT(ptes_evicted);
1677: pm->pm_evictions++;
1.12 matt 1678: /*
1679: * If this is a kernel page, make sure it's active.
1680: */
1681: if (pm == pmap_kernel()) {
1.45 thorpej 1682: i = pmap_pte_spill(pm, va, false);
1.12 matt 1683: KASSERT(i);
1684: }
1.1 matt 1685: }
1686: PMAP_PVO_CHECK(pvo); /* sanity check */
1687: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1688: pmap_pvo_enter_depth--;
1689: #endif
1690: pmap_interrupts_restore(msr);
1691: return 0;
1692: }
1693:
1.53 garbled 1694: static void
1.33 chs 1695: pmap_pvo_remove(struct pvo_entry *pvo, int pteidx, struct pvo_head *pvol)
1.1 matt 1696: {
1.2 matt 1697: volatile struct pte *pt;
1.1 matt 1698: int ptegidx;
1699:
1700: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1701: if (++pmap_pvo_remove_depth > 1)
1702: panic("pmap_pvo_remove: called recursively!");
1703: #endif
1704:
1705: /*
1706: * If we haven't been supplied the ptegidx, calculate it.
1707: */
1708: if (pteidx == -1) {
1.2 matt 1709: ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr);
1.1 matt 1710: pteidx = pmap_pvo_pte_index(pvo, ptegidx);
1711: } else {
1712: ptegidx = pteidx >> 3;
1713: if (pvo->pvo_pte.pte_hi & PTE_HID)
1714: ptegidx ^= pmap_pteg_mask;
1715: }
1716: PMAP_PVO_CHECK(pvo); /* sanity check */
1717:
1718: /*
1719: * If there is an active pte entry, we need to deactivate it
1720: * (and save the ref & chg bits).
1721: */
1722: pt = pmap_pvo_to_pte(pvo, pteidx);
1723: if (pt != NULL) {
1724: pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
1.12 matt 1725: PVO_WHERE(pvo, REMOVE);
1.1 matt 1726: PVO_PTEGIDX_CLR(pvo);
1727: PMAPCOUNT(ptes_removed);
1728: } else {
1729: KASSERT(pvo->pvo_pmap->pm_evictions > 0);
1730: pvo->pvo_pmap->pm_evictions--;
1731: }
1732:
1733: /*
1.14 chs 1734: * Account for executable mappings.
1735: */
1.39 matt 1736: if (PVO_EXECUTABLE_P(pvo))
1.14 chs 1737: pvo_clear_exec(pvo);
1738:
1739: /*
1740: * Update our statistics.
1.1 matt 1741: */
1742: pvo->pvo_pmap->pm_stats.resident_count--;
1.39 matt 1743: if (PVO_WIRED_P(pvo))
1.1 matt 1744: pvo->pvo_pmap->pm_stats.wired_count--;
1745:
1746: /*
1.107 chs 1747: * If the page is managed:
1748: * Save the REF/CHG bits into their cache.
1749: * Remove the PVO from the P/V list.
1.1 matt 1750: */
1.39 matt 1751: if (PVO_MANAGED_P(pvo)) {
1.2 matt 1752: register_t ptelo = pvo->pvo_pte.pte_lo;
1.1 matt 1753: struct vm_page *pg = PHYS_TO_VM_PAGE(ptelo & PTE_RPGN);
1754:
1755: if (pg != NULL) {
1.37 matt 1756: /*
1757: * If this page was changed and it is mapped exec,
1758: * invalidate it.
1759: */
1760: if ((ptelo & PTE_CHG) &&
1761: (pmap_attr_fetch(pg) & PTE_EXEC)) {
1762: struct pvo_head *pvoh = vm_page_to_pvoh(pg);
1763: if (LIST_EMPTY(pvoh)) {
1.85 matt 1764: DPRINTFN(EXEC, "[pmap_pvo_remove: "
1.53 garbled 1765: "%#" _PRIxpa ": clear-exec]\n",
1.85 matt 1766: VM_PAGE_TO_PHYS(pg));
1.37 matt 1767: pmap_attr_clear(pg, PTE_EXEC);
1768: PMAPCOUNT(exec_uncached_pvo_remove);
1769: } else {
1.85 matt 1770: DPRINTFN(EXEC, "[pmap_pvo_remove: "
1.53 garbled 1771: "%#" _PRIxpa ": syncicache]\n",
1.85 matt 1772: VM_PAGE_TO_PHYS(pg));
1.37 matt 1773: pmap_syncicache(VM_PAGE_TO_PHYS(pg),
1774: PAGE_SIZE);
1775: PMAPCOUNT(exec_synced_pvo_remove);
1776: }
1777: }
1778:
1.1 matt 1779: pmap_attr_save(pg, ptelo & (PTE_REF|PTE_CHG));
1780: }
1.107 chs 1781: LIST_REMOVE(pvo, pvo_vlink);
1.1 matt 1782: PMAPCOUNT(unmappings);
1783: } else {
1784: PMAPCOUNT(kernel_unmappings);
1785: }
1786:
1787: /*
1.107 chs 1788: * Remove the PVO from its list and return it to the pool.
1.1 matt 1789: */
1790: TAILQ_REMOVE(&pmap_pvo_table[ptegidx], pvo, pvo_olink);
1.33 chs 1791: if (pvol) {
1792: LIST_INSERT_HEAD(pvol, pvo, pvo_vlink);
1.25 chs 1793: }
1.1 matt 1794: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1795: pmap_pvo_remove_depth--;
1796: #endif
1797: }
1798:
1.33 chs 1799: void
1800: pmap_pvo_free(struct pvo_entry *pvo)
1801: {
1802:
1.106 martin 1803: pool_put(&pmap_pvo_pool, pvo);
1.33 chs 1804: }
1805:
1806: void
1807: pmap_pvo_free_list(struct pvo_head *pvol)
1808: {
1809: struct pvo_entry *pvo, *npvo;
1810:
1811: for (pvo = LIST_FIRST(pvol); pvo != NULL; pvo = npvo) {
1812: npvo = LIST_NEXT(pvo, pvo_vlink);
1813: LIST_REMOVE(pvo, pvo_vlink);
1814: pmap_pvo_free(pvo);
1815: }
1816: }
1817:
1.1 matt 1818: /*
1.14 chs 1819: * Mark a mapping as executable.
1820: * If this is the first executable mapping in the segment,
1821: * clear the noexec flag.
1822: */
1.53 garbled 1823: static void
1.14 chs 1824: pvo_set_exec(struct pvo_entry *pvo)
1825: {
1826: struct pmap *pm = pvo->pvo_pmap;
1827:
1.39 matt 1828: if (pm == pmap_kernel() || PVO_EXECUTABLE_P(pvo)) {
1.14 chs 1829: return;
1830: }
1831: pvo->pvo_vaddr |= PVO_EXECUTABLE;
1.53 garbled 1832: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.18 matt 1833: {
1834: int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT;
1835: if (pm->pm_exec[sr]++ == 0) {
1836: pm->pm_sr[sr] &= ~SR_NOEXEC;
1837: }
1.14 chs 1838: }
1.18 matt 1839: #endif
1.14 chs 1840: }
1841:
1842: /*
1843: * Mark a mapping as non-executable.
1844: * If this was the last executable mapping in the segment,
1845: * set the noexec flag.
1846: */
1.53 garbled 1847: static void
1.14 chs 1848: pvo_clear_exec(struct pvo_entry *pvo)
1849: {
1850: struct pmap *pm = pvo->pvo_pmap;
1851:
1.39 matt 1852: if (pm == pmap_kernel() || !PVO_EXECUTABLE_P(pvo)) {
1.14 chs 1853: return;
1854: }
1855: pvo->pvo_vaddr &= ~PVO_EXECUTABLE;
1.53 garbled 1856: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.18 matt 1857: {
1858: int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT;
1859: if (--pm->pm_exec[sr] == 0) {
1860: pm->pm_sr[sr] |= SR_NOEXEC;
1861: }
1.14 chs 1862: }
1.18 matt 1863: #endif
1.14 chs 1864: }
1865:
1866: /*
1.1 matt 1867: * Insert physical page at pa into the given pmap at virtual address va.
1868: */
1869: int
1.65 cegger 1870: pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1.1 matt 1871: {
1872: struct mem_region *mp;
1873: struct pvo_head *pvo_head;
1874: struct vm_page *pg;
1.2 matt 1875: register_t pte_lo;
1.1 matt 1876: int error;
1877: u_int was_exec = 0;
1878:
1.50 ad 1879: PMAP_LOCK();
1880:
1.1 matt 1881: if (__predict_false(!pmap_initialized)) {
1.107 chs 1882: pvo_head = NULL;
1.1 matt 1883: pg = NULL;
1884: was_exec = PTE_EXEC;
1.107 chs 1885:
1.1 matt 1886: } else {
1887: pvo_head = pa_to_pvoh(pa, &pg);
1888: }
1889:
1890: DPRINTFN(ENTER,
1.85 matt 1891: "pmap_enter(%p, %#" _PRIxva ", %#" _PRIxpa ", 0x%x, 0x%x):",
1892: pm, va, pa, prot, flags);
1.1 matt 1893:
1894: /*
1895: * If this is a managed page, and it's the first reference to the
1896: * page clear the execness of the page. Otherwise fetch the execness.
1897: */
1898: if (pg != NULL)
1899: was_exec = pmap_attr_fetch(pg) & PTE_EXEC;
1900:
1.85 matt 1901: DPRINTFN(ENTER, " was_exec=%d", was_exec);
1.1 matt 1902:
1903: /*
1904: * Assume the page is cache inhibited and access is guarded unless
1905: * it's in our available memory array. If it is in the memory array,
1906: * asssume it's in memory coherent memory.
1907: */
1.77 macallan 1908: if (flags & PMAP_MD_PREFETCHABLE) {
1909: pte_lo = 0;
1910: } else
1911: pte_lo = PTE_G;
1912:
1.81 matt 1913: if ((flags & PMAP_NOCACHE) == 0) {
1.1 matt 1914: for (mp = mem; mp->size; mp++) {
1915: if (pa >= mp->start && pa < mp->start + mp->size) {
1916: pte_lo = PTE_M;
1917: break;
1918: }
1919: }
1.87 kiyohara 1920: #ifdef MULTIPROCESSOR
1921: if (((mfpvr() >> 16) & 0xffff) == MPC603e)
1922: pte_lo = PTE_M;
1923: #endif
1.77 macallan 1924: } else {
1925: pte_lo |= PTE_I;
1.1 matt 1926: }
1927:
1928: if (prot & VM_PROT_WRITE)
1929: pte_lo |= PTE_BW;
1930: else
1931: pte_lo |= PTE_BR;
1932:
1933: /*
1934: * If this was in response to a fault, "pre-fault" the PTE's
1935: * changed/referenced bit appropriately.
1936: */
1937: if (flags & VM_PROT_WRITE)
1938: pte_lo |= PTE_CHG;
1.30 chs 1939: if (flags & VM_PROT_ALL)
1.1 matt 1940: pte_lo |= PTE_REF;
1941:
1942: /*
1943: * We need to know if this page can be executable
1944: */
1945: flags |= (prot & VM_PROT_EXECUTE);
1946:
1947: /*
1948: * Record mapping for later back-translation and pte spilling.
1949: * This will overwrite any existing mapping.
1950: */
1.106 martin 1951: error = pmap_pvo_enter(pm, &pmap_pvo_pool, pvo_head, va, pa, pte_lo, flags);
1.1 matt 1952:
1953: /*
1954: * Flush the real page from the instruction cache if this page is
1955: * mapped executable and cacheable and has not been flushed since
1956: * the last time it was modified.
1957: */
1958: if (error == 0 &&
1959: (flags & VM_PROT_EXECUTE) &&
1960: (pte_lo & PTE_I) == 0 &&
1961: was_exec == 0) {
1.85 matt 1962: DPRINTFN(ENTER, " %s", "syncicache");
1.1 matt 1963: PMAPCOUNT(exec_synced);
1.6 thorpej 1964: pmap_syncicache(pa, PAGE_SIZE);
1.1 matt 1965: if (pg != NULL) {
1966: pmap_attr_save(pg, PTE_EXEC);
1967: PMAPCOUNT(exec_cached);
1968: #if defined(DEBUG) || defined(PMAPDEBUG)
1969: if (pmapdebug & PMAPDEBUG_ENTER)
1970: printf(" marked-as-exec");
1971: else if (pmapdebug & PMAPDEBUG_EXEC)
1.53 garbled 1972: printf("[pmap_enter: %#" _PRIxpa ": marked-as-exec]\n",
1.34 yamt 1973: VM_PAGE_TO_PHYS(pg));
1.1 matt 1974: #endif
1975: }
1976: }
1977:
1.85 matt 1978: DPRINTFN(ENTER, ": error=%d\n", error);
1.1 matt 1979:
1.50 ad 1980: PMAP_UNLOCK();
1981:
1.1 matt 1982: return error;
1983: }
1984:
1985: void
1.68 cegger 1986: pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1.1 matt 1987: {
1988: struct mem_region *mp;
1.2 matt 1989: register_t pte_lo;
1.1 matt 1990: int error;
1991:
1.85 matt 1992: #if defined (PMAP_OEA64_BRIDGE) || defined (PMAP_OEA)
1.1 matt 1993: if (va < VM_MIN_KERNEL_ADDRESS)
1994: panic("pmap_kenter_pa: attempt to enter "
1.53 garbled 1995: "non-kernel address %#" _PRIxva "!", va);
1.38 sanjayl 1996: #endif
1.1 matt 1997:
1998: DPRINTFN(KENTER,
1.85 matt 1999: "pmap_kenter_pa(%#" _PRIxva ",%#" _PRIxpa ",%#x)\n", va, pa, prot);
1.1 matt 2000:
1.50 ad 2001: PMAP_LOCK();
2002:
1.1 matt 2003: /*
2004: * Assume the page is cache inhibited and access is guarded unless
2005: * it's in our available memory array. If it is in the memory array,
2006: * asssume it's in memory coherent memory.
2007: */
2008: pte_lo = PTE_IG;
1.81 matt 2009: if ((flags & PMAP_NOCACHE) == 0) {
1.4 matt 2010: for (mp = mem; mp->size; mp++) {
2011: if (pa >= mp->start && pa < mp->start + mp->size) {
2012: pte_lo = PTE_M;
2013: break;
2014: }
1.1 matt 2015: }
1.87 kiyohara 2016: #ifdef MULTIPROCESSOR
2017: if (((mfpvr() >> 16) & 0xffff) == MPC603e)
2018: pte_lo = PTE_M;
2019: #endif
1.1 matt 2020: }
2021:
2022: if (prot & VM_PROT_WRITE)
2023: pte_lo |= PTE_BW;
2024: else
2025: pte_lo |= PTE_BR;
2026:
2027: /*
2028: * We don't care about REF/CHG on PVOs on the unmanaged list.
2029: */
1.106 martin 2030: error = pmap_pvo_enter(pmap_kernel(), &pmap_pvo_pool,
1.107 chs 2031: NULL, va, pa, pte_lo, prot|PMAP_WIRED);
1.1 matt 2032:
2033: if (error != 0)
1.53 garbled 2034: panic("pmap_kenter_pa: failed to enter va %#" _PRIxva " pa %#" _PRIxpa ": %d",
1.1 matt 2035: va, pa, error);
1.50 ad 2036:
2037: PMAP_UNLOCK();
1.1 matt 2038: }
2039:
2040: void
2041: pmap_kremove(vaddr_t va, vsize_t len)
2042: {
2043: if (va < VM_MIN_KERNEL_ADDRESS)
2044: panic("pmap_kremove: attempt to remove "
1.53 garbled 2045: "non-kernel address %#" _PRIxva "!", va);
1.1 matt 2046:
1.85 matt 2047: DPRINTFN(KREMOVE, "pmap_kremove(%#" _PRIxva ",%#" _PRIxva ")\n", va, len);
1.1 matt 2048: pmap_remove(pmap_kernel(), va, va + len);
2049: }
2050:
2051: /*
2052: * Remove the given range of mapping entries.
2053: */
2054: void
2055: pmap_remove(pmap_t pm, vaddr_t va, vaddr_t endva)
2056: {
1.33 chs 2057: struct pvo_head pvol;
1.1 matt 2058: struct pvo_entry *pvo;
1.2 matt 2059: register_t msr;
1.1 matt 2060: int pteidx;
2061:
1.50 ad 2062: PMAP_LOCK();
1.33 chs 2063: LIST_INIT(&pvol);
1.14 chs 2064: msr = pmap_interrupts_off();
1.1 matt 2065: for (; va < endva; va += PAGE_SIZE) {
2066: pvo = pmap_pvo_find_va(pm, va, &pteidx);
2067: if (pvo != NULL) {
1.33 chs 2068: pmap_pvo_remove(pvo, pteidx, &pvol);
1.1 matt 2069: }
2070: }
1.14 chs 2071: pmap_interrupts_restore(msr);
1.33 chs 2072: pmap_pvo_free_list(&pvol);
1.50 ad 2073: PMAP_UNLOCK();
1.1 matt 2074: }
2075:
1.104 thorpej 2076: #if defined(PMAP_OEA)
2077: #ifdef PPC_OEA601
2078: bool
2079: pmap_extract_ioseg601(vaddr_t va, paddr_t *pap)
2080: {
2081: if ((MFPVR() >> 16) != MPC601)
2082: return false;
2083:
2084: const register_t sr = iosrtable[va >> ADDR_SR_SHFT];
2085:
2086: if (SR601_VALID_P(sr) && SR601_PA_MATCH_P(sr, va)) {
2087: if (pap)
2088: *pap = va;
2089: return true;
2090: }
2091: return false;
2092: }
2093:
2094: static bool
2095: pmap_extract_battable601(vaddr_t va, paddr_t *pap)
2096: {
2097: const register_t batu = battable[va >> 23].batu;
2098: const register_t batl = battable[va >> 23].batl;
2099:
2100: if (BAT601_VALID_P(batl) && BAT601_VA_MATCH_P(batu, batl, va)) {
2101: const register_t mask =
2102: (~(batl & BAT601_BSM) << 17) & ~0x1ffffL;
2103: if (pap)
2104: *pap = (batl & mask) | (va & ~mask);
2105: return true;
2106: }
2107: return false;
2108: }
2109: #endif /* PPC_OEA601 */
2110:
2111: bool
2112: pmap_extract_battable(vaddr_t va, paddr_t *pap)
2113: {
2114: #ifdef PPC_OEA601
2115: if ((MFPVR() >> 16) == MPC601)
2116: return pmap_extract_battable601(va, pap);
2117: #endif /* PPC_OEA601 */
2118:
2119: if (oeacpufeat & OEACPU_NOBAT)
2120: return false;
2121:
2122: const register_t batu = battable[BAT_VA2IDX(va)].batu;
2123:
2124: if (BAT_VALID_P(batu, 0) && BAT_VA_MATCH_P(batu, va)) {
2125: const register_t batl = battable[BAT_VA2IDX(va)].batl;
2126: const register_t mask =
2127: (~(batu & (BAT_XBL|BAT_BL)) << 15) & ~0x1ffffL;
2128: if (pap)
2129: *pap = (batl & mask) | (va & ~mask);
2130: return true;
2131: }
2132: return false;
2133: }
2134: #endif /* PMAP_OEA */
2135:
1.1 matt 2136: /*
2137: * Get the physical page address for the given pmap/virtual address.
2138: */
1.44 thorpej 2139: bool
1.1 matt 2140: pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap)
2141: {
2142: struct pvo_entry *pvo;
1.2 matt 2143: register_t msr;
1.7 matt 2144:
1.50 ad 2145: PMAP_LOCK();
1.38 sanjayl 2146:
1.7 matt 2147: /*
1.104 thorpej 2148: * If this is the kernel pmap, check the battable and I/O
2149: * segments for a hit. This is done only for regions outside
2150: * VM_MIN_KERNEL_ADDRESS-VM_MAX_KERNEL_ADDRESS.
2151: *
2152: * Be careful when checking VM_MAX_KERNEL_ADDRESS; you don't
2153: * want to wrap around to 0.
1.7 matt 2154: */
2155: if (pm == pmap_kernel() &&
2156: (va < VM_MIN_KERNEL_ADDRESS ||
2157: (KERNEL2_SR < 15 && VM_MAX_KERNEL_ADDRESS <= va))) {
1.8 matt 2158: KASSERT((va >> ADDR_SR_SHFT) != USER_SR);
1.104 thorpej 2159: #if defined(PMAP_OEA)
1.55 garbled 2160: #ifdef PPC_OEA601
1.104 thorpej 2161: if (pmap_extract_ioseg601(va, pap)) {
2162: PMAP_UNLOCK();
2163: return true;
2164: }
1.55 garbled 2165: #endif /* PPC_OEA601 */
1.104 thorpej 2166: if (pmap_extract_battable(va, pap)) {
2167: PMAP_UNLOCK();
2168: return true;
1.7 matt 2169: }
1.104 thorpej 2170: /*
2171: * We still check the HTAB...
2172: */
2173: #elif defined(PMAP_OEA64_BRIDGE)
2174: if (va < SEGMENT_LENGTH) {
2175: if (pap)
2176: *pap = va;
1.52 garbled 2177: PMAP_UNLOCK();
2178: return true;
1.104 thorpej 2179: }
2180: /*
2181: * We still check the HTAB...
2182: */
2183: #elif defined(PMAP_OEA64)
1.38 sanjayl 2184: #error PPC_OEA64 not supported
2185: #endif /* PPC_OEA */
1.7 matt 2186: }
1.1 matt 2187:
2188: msr = pmap_interrupts_off();
2189: pvo = pmap_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
2190: if (pvo != NULL) {
2191: PMAP_PVO_CHECK(pvo); /* sanity check */
1.29 briggs 2192: if (pap)
2193: *pap = (pvo->pvo_pte.pte_lo & PTE_RPGN)
2194: | (va & ADDR_POFF);
1.1 matt 2195: }
2196: pmap_interrupts_restore(msr);
1.50 ad 2197: PMAP_UNLOCK();
1.1 matt 2198: return pvo != NULL;
2199: }
2200:
2201: /*
2202: * Lower the protection on the specified range of this pmap.
2203: */
2204: void
2205: pmap_protect(pmap_t pm, vaddr_t va, vaddr_t endva, vm_prot_t prot)
2206: {
2207: struct pvo_entry *pvo;
1.2 matt 2208: volatile struct pte *pt;
2209: register_t msr;
1.1 matt 2210: int pteidx;
2211:
2212: /*
2213: * Since this routine only downgrades protection, we should
1.14 chs 2214: * always be called with at least one bit not set.
1.1 matt 2215: */
1.14 chs 2216: KASSERT(prot != VM_PROT_ALL);
1.1 matt 2217:
2218: /*
2219: * If there is no protection, this is equivalent to
2220: * remove the pmap from the pmap.
2221: */
2222: if ((prot & VM_PROT_READ) == 0) {
2223: pmap_remove(pm, va, endva);
2224: return;
2225: }
2226:
1.50 ad 2227: PMAP_LOCK();
2228:
1.1 matt 2229: msr = pmap_interrupts_off();
1.6 thorpej 2230: for (; va < endva; va += PAGE_SIZE) {
1.1 matt 2231: pvo = pmap_pvo_find_va(pm, va, &pteidx);
2232: if (pvo == NULL)
2233: continue;
2234: PMAP_PVO_CHECK(pvo); /* sanity check */
2235:
2236: /*
2237: * Revoke executable if asked to do so.
2238: */
2239: if ((prot & VM_PROT_EXECUTE) == 0)
1.14 chs 2240: pvo_clear_exec(pvo);
1.1 matt 2241:
2242: #if 0
2243: /*
2244: * If the page is already read-only, no change
2245: * needs to be made.
2246: */
2247: if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR)
2248: continue;
2249: #endif
2250: /*
2251: * Grab the PTE pointer before we diddle with
2252: * the cached PTE copy.
2253: */
2254: pt = pmap_pvo_to_pte(pvo, pteidx);
2255: /*
2256: * Change the protection of the page.
2257: */
2258: pvo->pvo_pte.pte_lo &= ~PTE_PP;
2259: pvo->pvo_pte.pte_lo |= PTE_BR;
2260:
2261: /*
2262: * If the PVO is in the page table, update
2263: * that pte at well.
2264: */
2265: if (pt != NULL) {
2266: pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
1.12 matt 2267: PVO_WHERE(pvo, PMAP_PROTECT);
1.1 matt 2268: PMAPCOUNT(ptes_changed);
2269: }
2270:
2271: PMAP_PVO_CHECK(pvo); /* sanity check */
2272: }
2273: pmap_interrupts_restore(msr);
1.50 ad 2274: PMAP_UNLOCK();
1.1 matt 2275: }
2276:
2277: void
2278: pmap_unwire(pmap_t pm, vaddr_t va)
2279: {
2280: struct pvo_entry *pvo;
1.2 matt 2281: register_t msr;
1.1 matt 2282:
1.50 ad 2283: PMAP_LOCK();
1.1 matt 2284: msr = pmap_interrupts_off();
2285: pvo = pmap_pvo_find_va(pm, va, NULL);
2286: if (pvo != NULL) {
1.39 matt 2287: if (PVO_WIRED_P(pvo)) {
1.1 matt 2288: pvo->pvo_vaddr &= ~PVO_WIRED;
2289: pm->pm_stats.wired_count--;
2290: }
2291: PMAP_PVO_CHECK(pvo); /* sanity check */
2292: }
2293: pmap_interrupts_restore(msr);
1.50 ad 2294: PMAP_UNLOCK();
1.1 matt 2295: }
2296:
1.108 riastrad 2297: static void
2298: pmap_pp_protect(struct pmap_page *pp, paddr_t pa, vm_prot_t prot)
1.1 matt 2299: {
1.33 chs 2300: struct pvo_head *pvo_head, pvol;
1.1 matt 2301: struct pvo_entry *pvo, *next_pvo;
1.2 matt 2302: volatile struct pte *pt;
2303: register_t msr;
1.1 matt 2304:
1.50 ad 2305: PMAP_LOCK();
2306:
1.14 chs 2307: KASSERT(prot != VM_PROT_ALL);
1.33 chs 2308: LIST_INIT(&pvol);
1.1 matt 2309: msr = pmap_interrupts_off();
2310:
2311: /*
2312: * When UVM reuses a page, it does a pmap_page_protect with
2313: * VM_PROT_NONE. At that point, we can clear the exec flag
2314: * since we know the page will have different contents.
2315: */
2316: if ((prot & VM_PROT_READ) == 0) {
1.85 matt 2317: DPRINTFN(EXEC, "[pmap_page_protect: %#" _PRIxpa ": clear-exec]\n",
1.108 riastrad 2318: pa);
2319: if (pmap_pp_attr_fetch(pp) & PTE_EXEC) {
1.1 matt 2320: PMAPCOUNT(exec_uncached_page_protect);
1.108 riastrad 2321: pmap_pp_attr_clear(pp, PTE_EXEC);
1.1 matt 2322: }
2323: }
2324:
1.108 riastrad 2325: pvo_head = &pp->pp_pvoh;
1.1 matt 2326: for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) {
2327: next_pvo = LIST_NEXT(pvo, pvo_vlink);
2328: PMAP_PVO_CHECK(pvo); /* sanity check */
2329:
2330: /*
2331: * Downgrading to no mapping at all, we just remove the entry.
2332: */
2333: if ((prot & VM_PROT_READ) == 0) {
1.33 chs 2334: pmap_pvo_remove(pvo, -1, &pvol);
1.1 matt 2335: continue;
2336: }
2337:
2338: /*
2339: * If EXEC permission is being revoked, just clear the
2340: * flag in the PVO.
2341: */
2342: if ((prot & VM_PROT_EXECUTE) == 0)
1.14 chs 2343: pvo_clear_exec(pvo);
1.1 matt 2344:
2345: /*
2346: * If this entry is already RO, don't diddle with the
2347: * page table.
2348: */
2349: if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) {
2350: PMAP_PVO_CHECK(pvo);
2351: continue;
2352: }
2353:
2354: /*
2355: * Grab the PTE before the we diddle the bits so
2356: * pvo_to_pte can verify the pte contents are as
2357: * expected.
2358: */
2359: pt = pmap_pvo_to_pte(pvo, -1);
2360: pvo->pvo_pte.pte_lo &= ~PTE_PP;
2361: pvo->pvo_pte.pte_lo |= PTE_BR;
2362: if (pt != NULL) {
2363: pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
1.12 matt 2364: PVO_WHERE(pvo, PMAP_PAGE_PROTECT);
1.1 matt 2365: PMAPCOUNT(ptes_changed);
2366: }
2367: PMAP_PVO_CHECK(pvo); /* sanity check */
2368: }
2369: pmap_interrupts_restore(msr);
1.33 chs 2370: pmap_pvo_free_list(&pvol);
1.50 ad 2371:
2372: PMAP_UNLOCK();
1.1 matt 2373: }
2374:
2375: /*
1.108 riastrad 2376: * Lower the protection on the specified physical page.
2377: */
2378: void
2379: pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
2380: {
2381: struct vm_page_md *md = VM_PAGE_TO_MD(pg);
2382:
2383: pmap_pp_protect(&md->mdpg_pp, VM_PAGE_TO_PHYS(pg), prot);
2384: }
2385:
2386: /*
2387: * Lower the protection on the physical page at the specified physical
2388: * address, which may not be managed and so may not have a struct
2389: * vm_page.
2390: */
2391: void
2392: pmap_pv_protect(paddr_t pa, vm_prot_t prot)
2393: {
2394: struct pmap_page *pp;
2395:
2396: if ((pp = pmap_pv_tracked(pa)) == NULL)
2397: return;
2398: pmap_pp_protect(pp, pa, prot);
2399: }
2400:
2401: /*
1.1 matt 2402: * Activate the address space for the specified process. If the process
2403: * is the current process, load the new MMU context.
2404: */
2405: void
2406: pmap_activate(struct lwp *l)
2407: {
1.69 rmind 2408: struct pcb *pcb = lwp_getpcb(l);
1.1 matt 2409: pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap;
2410:
2411: DPRINTFN(ACTIVATE,
1.85 matt 2412: "pmap_activate: lwp %p (curlwp %p)\n", l, curlwp);
1.1 matt 2413:
2414: /*
1.70 skrll 2415: * XXX Normally performed in cpu_lwp_fork().
1.1 matt 2416: */
1.13 matt 2417: pcb->pcb_pm = pmap;
1.17 matt 2418:
2419: /*
2420: * In theory, the SR registers need only be valid on return
2421: * to user space wait to do them there.
2422: */
2423: if (l == curlwp) {
2424: /* Store pointer to new current pmap. */
2425: curpm = pmap;
2426: }
1.1 matt 2427: }
2428:
2429: /*
2430: * Deactivate the specified process's address space.
2431: */
2432: void
2433: pmap_deactivate(struct lwp *l)
2434: {
2435: }
2436:
1.44 thorpej 2437: bool
1.1 matt 2438: pmap_query_bit(struct vm_page *pg, int ptebit)
2439: {
2440: struct pvo_entry *pvo;
1.2 matt 2441: volatile struct pte *pt;
2442: register_t msr;
1.1 matt 2443:
1.50 ad 2444: PMAP_LOCK();
2445:
2446: if (pmap_attr_fetch(pg) & ptebit) {
2447: PMAP_UNLOCK();
1.45 thorpej 2448: return true;
1.50 ad 2449: }
1.14 chs 2450:
1.1 matt 2451: msr = pmap_interrupts_off();
2452: LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) {
2453: PMAP_PVO_CHECK(pvo); /* sanity check */
2454: /*
2455: * See if we saved the bit off. If so cache, it and return
2456: * success.
2457: */
2458: if (pvo->pvo_pte.pte_lo & ptebit) {
2459: pmap_attr_save(pg, ptebit);
2460: PMAP_PVO_CHECK(pvo); /* sanity check */
2461: pmap_interrupts_restore(msr);
1.50 ad 2462: PMAP_UNLOCK();
1.45 thorpej 2463: return true;
1.1 matt 2464: }
2465: }
2466: /*
2467: * No luck, now go thru the hard part of looking at the ptes
2468: * themselves. Sync so any pending REF/CHG bits are flushed
2469: * to the PTEs.
2470: */
2471: SYNC();
2472: LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) {
2473: PMAP_PVO_CHECK(pvo); /* sanity check */
2474: /*
2475: * See if this pvo have a valid PTE. If so, fetch the
2476: * REF/CHG bits from the valid PTE. If the appropriate
2477: * ptebit is set, cache, it and return success.
2478: */
2479: pt = pmap_pvo_to_pte(pvo, -1);
2480: if (pt != NULL) {
2481: pmap_pte_synch(pt, &pvo->pvo_pte);
2482: if (pvo->pvo_pte.pte_lo & ptebit) {
2483: pmap_attr_save(pg, ptebit);
2484: PMAP_PVO_CHECK(pvo); /* sanity check */
2485: pmap_interrupts_restore(msr);
1.50 ad 2486: PMAP_UNLOCK();
1.45 thorpej 2487: return true;
1.1 matt 2488: }
2489: }
2490: }
2491: pmap_interrupts_restore(msr);
1.50 ad 2492: PMAP_UNLOCK();
1.45 thorpej 2493: return false;
1.1 matt 2494: }
2495:
1.44 thorpej 2496: bool
1.1 matt 2497: pmap_clear_bit(struct vm_page *pg, int ptebit)
2498: {
2499: struct pvo_head *pvoh = vm_page_to_pvoh(pg);
2500: struct pvo_entry *pvo;
1.2 matt 2501: volatile struct pte *pt;
2502: register_t msr;
1.1 matt 2503: int rv = 0;
2504:
1.50 ad 2505: PMAP_LOCK();
1.1 matt 2506: msr = pmap_interrupts_off();
2507:
2508: /*
2509: * Fetch the cache value
2510: */
2511: rv |= pmap_attr_fetch(pg);
2512:
2513: /*
2514: * Clear the cached value.
2515: */
2516: pmap_attr_clear(pg, ptebit);
2517:
2518: /*
2519: * Sync so any pending REF/CHG bits are flushed to the PTEs (so we
2520: * can reset the right ones). Note that since the pvo entries and
2521: * list heads are accessed via BAT0 and are never placed in the
2522: * page table, we don't have to worry about further accesses setting
2523: * the REF/CHG bits.
2524: */
2525: SYNC();
2526:
2527: /*
2528: * For each pvo entry, clear pvo's ptebit. If this pvo have a
2529: * valid PTE. If so, clear the ptebit from the valid PTE.
2530: */
2531: LIST_FOREACH(pvo, pvoh, pvo_vlink) {
2532: PMAP_PVO_CHECK(pvo); /* sanity check */
2533: pt = pmap_pvo_to_pte(pvo, -1);
2534: if (pt != NULL) {
2535: /*
2536: * Only sync the PTE if the bit we are looking
2537: * for is not already set.
2538: */
2539: if ((pvo->pvo_pte.pte_lo & ptebit) == 0)
2540: pmap_pte_synch(pt, &pvo->pvo_pte);
2541: /*
2542: * If the bit we are looking for was already set,
2543: * clear that bit in the pte.
2544: */
2545: if (pvo->pvo_pte.pte_lo & ptebit)
2546: pmap_pte_clear(pt, PVO_VADDR(pvo), ptebit);
2547: }
2548: rv |= pvo->pvo_pte.pte_lo & (PTE_CHG|PTE_REF);
2549: pvo->pvo_pte.pte_lo &= ~ptebit;
2550: PMAP_PVO_CHECK(pvo); /* sanity check */
2551: }
2552: pmap_interrupts_restore(msr);
1.14 chs 2553:
1.1 matt 2554: /*
2555: * If we are clearing the modify bit and this page was marked EXEC
2556: * and the user of the page thinks the page was modified, then we
2557: * need to clean it from the icache if it's mapped or clear the EXEC
2558: * bit if it's not mapped. The page itself might not have the CHG
2559: * bit set if the modification was done via DMA to the page.
2560: */
2561: if ((ptebit & PTE_CHG) && (rv & PTE_EXEC)) {
2562: if (LIST_EMPTY(pvoh)) {
1.85 matt 2563: DPRINTFN(EXEC, "[pmap_clear_bit: %#" _PRIxpa ": clear-exec]\n",
2564: VM_PAGE_TO_PHYS(pg));
1.1 matt 2565: pmap_attr_clear(pg, PTE_EXEC);
2566: PMAPCOUNT(exec_uncached_clear_modify);
2567: } else {
1.85 matt 2568: DPRINTFN(EXEC, "[pmap_clear_bit: %#" _PRIxpa ": syncicache]\n",
2569: VM_PAGE_TO_PHYS(pg));
1.34 yamt 2570: pmap_syncicache(VM_PAGE_TO_PHYS(pg), PAGE_SIZE);
1.1 matt 2571: PMAPCOUNT(exec_synced_clear_modify);
2572: }
2573: }
1.50 ad 2574: PMAP_UNLOCK();
1.1 matt 2575: return (rv & ptebit) != 0;
2576: }
2577:
2578: void
2579: pmap_procwr(struct proc *p, vaddr_t va, size_t len)
2580: {
2581: struct pvo_entry *pvo;
2582: size_t offset = va & ADDR_POFF;
2583: int s;
2584:
1.50 ad 2585: PMAP_LOCK();
1.1 matt 2586: s = splvm();
2587: while (len > 0) {
1.6 thorpej 2588: size_t seglen = PAGE_SIZE - offset;
1.1 matt 2589: if (seglen > len)
2590: seglen = len;
2591: pvo = pmap_pvo_find_va(p->p_vmspace->vm_map.pmap, va, NULL);
1.39 matt 2592: if (pvo != NULL && PVO_EXECUTABLE_P(pvo)) {
1.1 matt 2593: pmap_syncicache(
2594: (pvo->pvo_pte.pte_lo & PTE_RPGN) | offset, seglen);
2595: PMAP_PVO_CHECK(pvo);
2596: }
2597: va += seglen;
2598: len -= seglen;
2599: offset = 0;
2600: }
2601: splx(s);
1.50 ad 2602: PMAP_UNLOCK();
1.1 matt 2603: }
2604:
2605: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
2606: void
1.2 matt 2607: pmap_pte_print(volatile struct pte *pt)
1.1 matt 2608: {
2609: printf("PTE %p: ", pt);
1.38 sanjayl 2610:
1.53 garbled 2611: #if defined(PMAP_OEA)
1.1 matt 2612: /* High word: */
1.54 mlelstv 2613: printf("%#" _PRIxpte ": [", pt->pte_hi);
1.53 garbled 2614: #else
1.54 mlelstv 2615: printf("%#" _PRIxpte ": [", pt->pte_hi);
1.53 garbled 2616: #endif /* PMAP_OEA */
1.38 sanjayl 2617:
1.1 matt 2618: printf("%c ", (pt->pte_hi & PTE_VALID) ? 'v' : 'i');
2619: printf("%c ", (pt->pte_hi & PTE_HID) ? 'h' : '-');
1.38 sanjayl 2620:
1.54 mlelstv 2621: printf("%#" _PRIxpte " %#" _PRIxpte "",
1.38 sanjayl 2622: (pt->pte_hi &~ PTE_VALID)>>PTE_VSID_SHFT,
2623: pt->pte_hi & PTE_API);
1.53 garbled 2624: #if defined(PMAP_OEA) || defined(PMAP_OEA64_BRIDGE)
1.54 mlelstv 2625: printf(" (va %#" _PRIxva ")] ", pmap_pte_to_va(pt));
1.38 sanjayl 2626: #else
1.54 mlelstv 2627: printf(" (va %#" _PRIxva ")] ", pmap_pte_to_va(pt));
1.53 garbled 2628: #endif /* PMAP_OEA */
1.38 sanjayl 2629:
1.1 matt 2630: /* Low word: */
1.53 garbled 2631: #if defined (PMAP_OEA)
1.54 mlelstv 2632: printf(" %#" _PRIxpte ": [", pt->pte_lo);
2633: printf("%#" _PRIxpte "... ", pt->pte_lo >> 12);
1.53 garbled 2634: #else
1.54 mlelstv 2635: printf(" %#" _PRIxpte ": [", pt->pte_lo);
2636: printf("%#" _PRIxpte "... ", pt->pte_lo >> 12);
1.38 sanjayl 2637: #endif
1.1 matt 2638: printf("%c ", (pt->pte_lo & PTE_REF) ? 'r' : 'u');
2639: printf("%c ", (pt->pte_lo & PTE_CHG) ? 'c' : 'n');
2640: printf("%c", (pt->pte_lo & PTE_W) ? 'w' : '.');
2641: printf("%c", (pt->pte_lo & PTE_I) ? 'i' : '.');
2642: printf("%c", (pt->pte_lo & PTE_M) ? 'm' : '.');
2643: printf("%c ", (pt->pte_lo & PTE_G) ? 'g' : '.');
2644: switch (pt->pte_lo & PTE_PP) {
2645: case PTE_BR: printf("br]\n"); break;
2646: case PTE_BW: printf("bw]\n"); break;
2647: case PTE_SO: printf("so]\n"); break;
2648: case PTE_SW: printf("sw]\n"); break;
2649: }
2650: }
2651: #endif
2652:
2653: #if defined(DDB)
2654: void
2655: pmap_pteg_check(void)
2656: {
1.2 matt 2657: volatile struct pte *pt;
1.1 matt 2658: int i;
2659: int ptegidx;
2660: u_int p_valid = 0;
2661: u_int s_valid = 0;
2662: u_int invalid = 0;
1.38 sanjayl 2663:
1.1 matt 2664: for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) {
2665: for (pt = pmap_pteg_table[ptegidx].pt, i = 8; --i >= 0; pt++) {
2666: if (pt->pte_hi & PTE_VALID) {
2667: if (pt->pte_hi & PTE_HID)
2668: s_valid++;
2669: else
1.38 sanjayl 2670: {
1.1 matt 2671: p_valid++;
1.38 sanjayl 2672: }
1.1 matt 2673: } else
2674: invalid++;
2675: }
2676: }
2677: printf("pteg_check: v(p) %#x (%d), v(s) %#x (%d), i %#x (%d)\n",
2678: p_valid, p_valid, s_valid, s_valid,
2679: invalid, invalid);
2680: }
2681:
2682: void
2683: pmap_print_mmuregs(void)
2684: {
2685: int i;
1.97 rin 2686: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1 matt 2687: u_int cpuvers;
1.90 mrg 2688: #endif
1.53 garbled 2689: #ifndef PMAP_OEA64
1.1 matt 2690: vaddr_t addr;
1.2 matt 2691: register_t soft_sr[16];
1.18 matt 2692: #endif
1.97 rin 2693: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1 matt 2694: struct bat soft_ibat[4];
2695: struct bat soft_dbat[4];
1.38 sanjayl 2696: #endif
1.53 garbled 2697: paddr_t sdr1;
1.1 matt 2698:
1.97 rin 2699: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1 matt 2700: cpuvers = MFPVR() >> 16;
1.90 mrg 2701: #endif
1.35 perry 2702: __asm volatile ("mfsdr1 %0" : "=r"(sdr1));
1.53 garbled 2703: #ifndef PMAP_OEA64
1.16 kleink 2704: addr = 0;
1.27 chs 2705: for (i = 0; i < 16; i++) {
1.1 matt 2706: soft_sr[i] = MFSRIN(addr);
2707: addr += (1 << ADDR_SR_SHFT);
2708: }
1.18 matt 2709: #endif
1.1 matt 2710:
1.97 rin 2711: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1 matt 2712: /* read iBAT (601: uBAT) registers */
1.35 perry 2713: __asm volatile ("mfibatu %0,0" : "=r"(soft_ibat[0].batu));
2714: __asm volatile ("mfibatl %0,0" : "=r"(soft_ibat[0].batl));
2715: __asm volatile ("mfibatu %0,1" : "=r"(soft_ibat[1].batu));
2716: __asm volatile ("mfibatl %0,1" : "=r"(soft_ibat[1].batl));
2717: __asm volatile ("mfibatu %0,2" : "=r"(soft_ibat[2].batu));
2718: __asm volatile ("mfibatl %0,2" : "=r"(soft_ibat[2].batl));
2719: __asm volatile ("mfibatu %0,3" : "=r"(soft_ibat[3].batu));
2720: __asm volatile ("mfibatl %0,3" : "=r"(soft_ibat[3].batl));
1.1 matt 2721:
2722:
2723: if (cpuvers != MPC601) {
2724: /* read dBAT registers */
1.35 perry 2725: __asm volatile ("mfdbatu %0,0" : "=r"(soft_dbat[0].batu));
2726: __asm volatile ("mfdbatl %0,0" : "=r"(soft_dbat[0].batl));
2727: __asm volatile ("mfdbatu %0,1" : "=r"(soft_dbat[1].batu));
2728: __asm volatile ("mfdbatl %0,1" : "=r"(soft_dbat[1].batl));
2729: __asm volatile ("mfdbatu %0,2" : "=r"(soft_dbat[2].batu));
2730: __asm volatile ("mfdbatl %0,2" : "=r"(soft_dbat[2].batl));
2731: __asm volatile ("mfdbatu %0,3" : "=r"(soft_dbat[3].batu));
2732: __asm volatile ("mfdbatl %0,3" : "=r"(soft_dbat[3].batl));
1.1 matt 2733: }
1.38 sanjayl 2734: #endif
1.1 matt 2735:
1.54 mlelstv 2736: printf("SDR1:\t%#" _PRIxpa "\n", sdr1);
1.53 garbled 2737: #ifndef PMAP_OEA64
1.1 matt 2738: printf("SR[]:\t");
1.27 chs 2739: for (i = 0; i < 4; i++)
1.53 garbled 2740: printf("0x%08lx, ", soft_sr[i]);
1.1 matt 2741: printf("\n\t");
1.27 chs 2742: for ( ; i < 8; i++)
1.53 garbled 2743: printf("0x%08lx, ", soft_sr[i]);
1.1 matt 2744: printf("\n\t");
1.27 chs 2745: for ( ; i < 12; i++)
1.53 garbled 2746: printf("0x%08lx, ", soft_sr[i]);
1.1 matt 2747: printf("\n\t");
1.27 chs 2748: for ( ; i < 16; i++)
1.53 garbled 2749: printf("0x%08lx, ", soft_sr[i]);
1.1 matt 2750: printf("\n");
1.18 matt 2751: #endif
1.1 matt 2752:
1.97 rin 2753: #if defined(PMAP_OEA) || defined(PMAP_OEA64_BRIDGE)
1.1 matt 2754: printf("%cBAT[]:\t", cpuvers == MPC601 ? 'u' : 'i');
1.27 chs 2755: for (i = 0; i < 4; i++) {
1.2 matt 2756: printf("0x%08lx 0x%08lx, ",
1.1 matt 2757: soft_ibat[i].batu, soft_ibat[i].batl);
2758: if (i == 1)
2759: printf("\n\t");
2760: }
2761: if (cpuvers != MPC601) {
2762: printf("\ndBAT[]:\t");
1.27 chs 2763: for (i = 0; i < 4; i++) {
1.2 matt 2764: printf("0x%08lx 0x%08lx, ",
1.1 matt 2765: soft_dbat[i].batu, soft_dbat[i].batl);
2766: if (i == 1)
2767: printf("\n\t");
2768: }
2769: }
2770: printf("\n");
1.53 garbled 2771: #endif /* PMAP_OEA... */
1.1 matt 2772: }
2773:
2774: void
2775: pmap_print_pte(pmap_t pm, vaddr_t va)
2776: {
2777: struct pvo_entry *pvo;
1.2 matt 2778: volatile struct pte *pt;
1.1 matt 2779: int pteidx;
2780:
2781: pvo = pmap_pvo_find_va(pm, va, &pteidx);
2782: if (pvo != NULL) {
2783: pt = pmap_pvo_to_pte(pvo, pteidx);
2784: if (pt != NULL) {
1.53 garbled 2785: printf("VA %#" _PRIxva " -> %p -> %s %#" _PRIxpte ", %#" _PRIxpte "\n",
1.38 sanjayl 2786: va, pt,
2787: pt->pte_hi & PTE_HID ? "(sec)" : "(pri)",
2788: pt->pte_hi, pt->pte_lo);
1.1 matt 2789: } else {
2790: printf("No valid PTE found\n");
2791: }
2792: } else {
2793: printf("Address not in pmap\n");
2794: }
2795: }
2796:
2797: void
2798: pmap_pteg_dist(void)
2799: {
2800: struct pvo_entry *pvo;
2801: int ptegidx;
2802: int depth;
2803: int max_depth = 0;
2804: unsigned int depths[64];
2805:
2806: memset(depths, 0, sizeof(depths));
2807: for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) {
2808: depth = 0;
2809: TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
2810: depth++;
2811: }
2812: if (depth > max_depth)
2813: max_depth = depth;
2814: if (depth > 63)
2815: depth = 63;
2816: depths[depth]++;
2817: }
2818:
2819: for (depth = 0; depth < 64; depth++) {
2820: printf(" [%2d]: %8u", depth, depths[depth]);
2821: if ((depth & 3) == 3)
2822: printf("\n");
2823: if (depth == max_depth)
2824: break;
2825: }
2826: if ((depth & 3) != 3)
2827: printf("\n");
2828: printf("Max depth found was %d\n", max_depth);
2829: }
2830: #endif /* DEBUG */
2831:
2832: #if defined(PMAPCHECK) || defined(DEBUG)
2833: void
2834: pmap_pvo_verify(void)
2835: {
2836: int ptegidx;
2837: int s;
2838:
2839: s = splvm();
2840: for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) {
2841: struct pvo_entry *pvo;
2842: TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
2843: if ((uintptr_t) pvo >= SEGMENT_LENGTH)
2844: panic("pmap_pvo_verify: invalid pvo %p "
2845: "on list %#x", pvo, ptegidx);
2846: pmap_pvo_check(pvo);
2847: }
2848: }
2849: splx(s);
2850: }
2851: #endif /* PMAPCHECK */
2852:
2853: void *
1.106 martin 2854: pmap_pool_alloc(struct pool *pp, int flags)
1.1 matt 2855: {
2856: struct pvo_page *pvop;
1.106 martin 2857: struct vm_page *pg;
1.1 matt 2858:
1.50 ad 2859: if (uvm.page_init_done != true) {
2860: return (void *) uvm_pageboot_alloc(PAGE_SIZE);
2861: }
2862:
2863: PMAP_LOCK();
1.106 martin 2864: pvop = SIMPLEQ_FIRST(&pmap_pvop_head);
1.1 matt 2865: if (pvop != NULL) {
1.106 martin 2866: pmap_pvop_free--;
2867: SIMPLEQ_REMOVE_HEAD(&pmap_pvop_head, pvop_link);
1.50 ad 2868: PMAP_UNLOCK();
1.1 matt 2869: return pvop;
2870: }
1.50 ad 2871: PMAP_UNLOCK();
1.1 matt 2872: again:
2873: pg = uvm_pagealloc_strat(NULL, 0, NULL, UVM_PGA_USERESERVE,
2874: UVM_PGA_STRAT_ONLY, VM_FREELIST_FIRST256);
2875: if (__predict_false(pg == NULL)) {
2876: if (flags & PR_WAITOK) {
2877: uvm_wait("plpg");
2878: goto again;
2879: } else {
2880: return (0);
2881: }
2882: }
1.53 garbled 2883: KDASSERT(VM_PAGE_TO_PHYS(pg) == (uintptr_t)VM_PAGE_TO_PHYS(pg));
2884: return (void *)(uintptr_t) VM_PAGE_TO_PHYS(pg);
1.1 matt 2885: }
2886:
2887: void
1.106 martin 2888: pmap_pool_free(struct pool *pp, void *va)
1.1 matt 2889: {
2890: struct pvo_page *pvop;
2891:
1.50 ad 2892: PMAP_LOCK();
1.1 matt 2893: pvop = va;
1.106 martin 2894: SIMPLEQ_INSERT_HEAD(&pmap_pvop_head, pvop, pvop_link);
2895: pmap_pvop_free++;
2896: if (pmap_pvop_free > pmap_pvop_maxfree)
2897: pmap_pvop_maxfree = pmap_pvop_free;
1.50 ad 2898: PMAP_UNLOCK();
1.1 matt 2899: #if 0
2900: uvm_pagefree(PHYS_TO_VM_PAGE((paddr_t) va));
2901: #endif
2902: }
2903:
2904: /*
2905: * This routine in bootstraping to steal to-be-managed memory (which will
2906: * then be unmanaged). We use it to grab from the first 256MB for our
2907: * pmap needs and above 256MB for other stuff.
2908: */
2909: vaddr_t
1.10 thorpej 2910: pmap_steal_memory(vsize_t vsize, vaddr_t *vstartp, vaddr_t *vendp)
1.1 matt 2911: {
2912: vsize_t size;
2913: vaddr_t va;
1.94 cherry 2914: paddr_t start, end, pa = 0;
2915: int npgs, freelist;
2916: uvm_physseg_t bank;
1.1 matt 2917:
1.45 thorpej 2918: if (uvm.page_init_done == true)
1.1 matt 2919: panic("pmap_steal_memory: called _after_ bootstrap");
2920:
1.10 thorpej 2921: *vstartp = VM_MIN_KERNEL_ADDRESS;
2922: *vendp = VM_MAX_KERNEL_ADDRESS;
2923:
1.1 matt 2924: size = round_page(vsize);
2925: npgs = atop(size);
2926:
2927: /*
2928: * PA 0 will never be among those given to UVM so we can use it
2929: * to indicate we couldn't steal any memory.
2930: */
1.94 cherry 2931:
2932: for (bank = uvm_physseg_get_first();
2933: uvm_physseg_valid_p(bank);
2934: bank = uvm_physseg_get_next(bank)) {
2935:
2936: freelist = uvm_physseg_get_free_list(bank);
2937: start = uvm_physseg_get_start(bank);
2938: end = uvm_physseg_get_end(bank);
2939:
2940: if (freelist == VM_FREELIST_FIRST256 &&
2941: (end - start) >= npgs) {
2942: pa = ptoa(start);
1.1 matt 2943: break;
2944: }
2945: }
2946:
2947: if (pa == 0)
2948: panic("pmap_steal_memory: no approriate memory to steal!");
2949:
1.94 cherry 2950: uvm_physseg_unplug(start, npgs);
1.1 matt 2951:
2952: va = (vaddr_t) pa;
1.46 christos 2953: memset((void *) va, 0, size);
1.1 matt 2954: pmap_pages_stolen += npgs;
2955: #ifdef DEBUG
2956: if (pmapdebug && npgs > 1) {
2957: u_int cnt = 0;
1.94 cherry 2958: for (bank = uvm_physseg_get_first();
2959: uvm_physseg_valid_p(bank);
2960: bank = uvm_physseg_get_next(bank)) {
2961: cnt += uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank);
1.73 uebayasi 2962: }
1.1 matt 2963: printf("pmap_steal_memory: stole %u (total %u) pages (%u left)\n",
2964: npgs, pmap_pages_stolen, cnt);
2965: }
2966: #endif
2967:
2968: return va;
2969: }
2970:
2971: /*
2972: * Find a chuck of memory with right size and alignment.
2973: */
1.53 garbled 2974: paddr_t
1.1 matt 2975: pmap_boot_find_memory(psize_t size, psize_t alignment, int at_end)
2976: {
2977: struct mem_region *mp;
2978: paddr_t s, e;
2979: int i, j;
2980:
2981: size = round_page(size);
2982:
2983: DPRINTFN(BOOT,
1.85 matt 2984: "pmap_boot_find_memory: size=%#" _PRIxpa ", alignment=%#" _PRIxpa ", at_end=%d",
2985: size, alignment, at_end);
1.1 matt 2986:
1.6 thorpej 2987: if (alignment < PAGE_SIZE || (alignment & (alignment-1)) != 0)
1.54 mlelstv 2988: panic("pmap_boot_find_memory: invalid alignment %#" _PRIxpa,
1.1 matt 2989: alignment);
2990:
2991: if (at_end) {
1.6 thorpej 2992: if (alignment != PAGE_SIZE)
1.1 matt 2993: panic("pmap_boot_find_memory: invalid ending "
1.53 garbled 2994: "alignment %#" _PRIxpa, alignment);
1.1 matt 2995:
2996: for (mp = &avail[avail_cnt-1]; mp >= avail; mp--) {
2997: s = mp->start + mp->size - size;
2998: if (s >= mp->start && mp->size >= size) {
1.85 matt 2999: DPRINTFN(BOOT, ": %#" _PRIxpa "\n", s);
1.1 matt 3000: DPRINTFN(BOOT,
1.85 matt 3001: "pmap_boot_find_memory: b-avail[%d] start "
3002: "%#" _PRIxpa " size %#" _PRIxpa "\n", mp - avail,
3003: mp->start, mp->size);
1.1 matt 3004: mp->size -= size;
3005: DPRINTFN(BOOT,
1.85 matt 3006: "pmap_boot_find_memory: a-avail[%d] start "
3007: "%#" _PRIxpa " size %#" _PRIxpa "\n", mp - avail,
3008: mp->start, mp->size);
1.53 garbled 3009: return s;
1.1 matt 3010: }
3011: }
3012: panic("pmap_boot_find_memory: no available memory");
3013: }
3014:
3015: for (mp = avail, i = 0; i < avail_cnt; i++, mp++) {
3016: s = (mp->start + alignment - 1) & ~(alignment-1);
3017: e = s + size;
3018:
3019: /*
3020: * Is the calculated region entirely within the region?
3021: */
3022: if (s < mp->start || e > mp->start + mp->size)
3023: continue;
3024:
1.85 matt 3025: DPRINTFN(BOOT, ": %#" _PRIxpa "\n", s);
1.1 matt 3026: if (s == mp->start) {
3027: /*
3028: * If the block starts at the beginning of region,
3029: * adjust the size & start. (the region may now be
3030: * zero in length)
3031: */
3032: DPRINTFN(BOOT,
1.85 matt 3033: "pmap_boot_find_memory: b-avail[%d] start "
3034: "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1 matt 3035: mp->start += size;
3036: mp->size -= size;
3037: DPRINTFN(BOOT,
1.85 matt 3038: "pmap_boot_find_memory: a-avail[%d] start "
3039: "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1 matt 3040: } else if (e == mp->start + mp->size) {
3041: /*
3042: * If the block starts at the beginning of region,
3043: * adjust only the size.
3044: */
3045: DPRINTFN(BOOT,
1.85 matt 3046: "pmap_boot_find_memory: b-avail[%d] start "
3047: "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1 matt 3048: mp->size -= size;
3049: DPRINTFN(BOOT,
1.85 matt 3050: "pmap_boot_find_memory: a-avail[%d] start "
3051: "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1 matt 3052: } else {
3053: /*
3054: * Block is in the middle of the region, so we
3055: * have to split it in two.
3056: */
3057: for (j = avail_cnt; j > i + 1; j--) {
3058: avail[j] = avail[j-1];
3059: }
3060: DPRINTFN(BOOT,
1.85 matt 3061: "pmap_boot_find_memory: b-avail[%d] start "
3062: "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1 matt 3063: mp[1].start = e;
3064: mp[1].size = mp[0].start + mp[0].size - e;
3065: mp[0].size = s - mp[0].start;
3066: avail_cnt++;
3067: for (; i < avail_cnt; i++) {
3068: DPRINTFN(BOOT,
1.85 matt 3069: "pmap_boot_find_memory: a-avail[%d] "
3070: "start %#" _PRIxpa " size %#" _PRIxpa "\n", i,
3071: avail[i].start, avail[i].size);
1.1 matt 3072: }
3073: }
1.53 garbled 3074: KASSERT(s == (uintptr_t) s);
3075: return s;
1.1 matt 3076: }
3077: panic("pmap_boot_find_memory: not enough memory for "
1.54 mlelstv 3078: "%#" _PRIxpa "/%#" _PRIxpa " allocation?", size, alignment);
1.1 matt 3079: }
3080:
1.38 sanjayl 3081: /* XXXSL: we dont have any BATs to do this, map in Segment 0 1:1 using page tables */
1.53 garbled 3082: #if defined (PMAP_OEA64_BRIDGE)
1.38 sanjayl 3083: int
3084: pmap_setup_segment0_map(int use_large_pages, ...)
3085: {
1.88 christos 3086: vaddr_t va, va_end;
1.38 sanjayl 3087:
3088: register_t pte_lo = 0x0;
1.90 mrg 3089: int ptegidx = 0;
1.38 sanjayl 3090: struct pte pte;
3091: va_list ap;
3092:
3093: /* Coherent + Supervisor RW, no user access */
3094: pte_lo = PTE_M;
3095:
3096: /* XXXSL
3097: * Map in 1st segment 1:1, we'll be careful not to spill kernel entries later,
3098: * these have to take priority.
3099: */
3100: for (va = 0x0; va < SEGMENT_LENGTH; va += 0x1000) {
3101: ptegidx = va_to_pteg(pmap_kernel(), va);
3102: pmap_pte_create(&pte, pmap_kernel(), va, va | pte_lo);
1.90 mrg 3103: (void)pmap_pte_insert(ptegidx, &pte);
1.38 sanjayl 3104: }
3105:
3106: va_start(ap, use_large_pages);
3107: while (1) {
3108: paddr_t pa;
3109: size_t size;
3110:
3111: va = va_arg(ap, vaddr_t);
3112:
3113: if (va == 0)
3114: break;
3115:
3116: pa = va_arg(ap, paddr_t);
3117: size = va_arg(ap, size_t);
3118:
1.88 christos 3119: for (va_end = va + size; va < va_end; va += 0x1000, pa += 0x1000) {
1.38 sanjayl 3120: #if 0
1.54 mlelstv 3121: printf("%s: Inserting: va: %#" _PRIxva ", pa: %#" _PRIxpa "\n", __func__, va, pa);
1.38 sanjayl 3122: #endif
3123: ptegidx = va_to_pteg(pmap_kernel(), va);
3124: pmap_pte_create(&pte, pmap_kernel(), va, pa | pte_lo);
1.90 mrg 3125: (void)pmap_pte_insert(ptegidx, &pte);
1.38 sanjayl 3126: }
3127: }
1.93 dholland 3128: va_end(ap);
1.38 sanjayl 3129:
3130: TLBSYNC();
3131: SYNC();
3132: return (0);
3133: }
1.53 garbled 3134: #endif /* PMAP_OEA64_BRIDGE */
1.38 sanjayl 3135:
1.1 matt 3136: /*
1.99 thorpej 3137: * Set up the bottom level of the data structures necessary for the kernel
3138: * to manage memory. MMU hardware is programmed in pmap_bootstrap2().
1.1 matt 3139: */
3140: void
1.99 thorpej 3141: pmap_bootstrap1(paddr_t kernelstart, paddr_t kernelend)
1.1 matt 3142: {
3143: struct mem_region *mp, tmp;
3144: paddr_t s, e;
3145: psize_t size;
3146: int i, j;
3147:
3148: /*
3149: * Get memory.
3150: */
3151: mem_regions(&mem, &avail);
3152: #if defined(DEBUG)
3153: if (pmapdebug & PMAPDEBUG_BOOT) {
3154: printf("pmap_bootstrap: memory configuration:\n");
3155: for (mp = mem; mp->size; mp++) {
1.54 mlelstv 3156: printf("pmap_bootstrap: mem start %#" _PRIxpa " size %#" _PRIxpa "\n",
1.1 matt 3157: mp->start, mp->size);
3158: }
3159: for (mp = avail; mp->size; mp++) {
1.54 mlelstv 3160: printf("pmap_bootstrap: avail start %#" _PRIxpa " size %#" _PRIxpa "\n",
1.1 matt 3161: mp->start, mp->size);
3162: }
3163: }
3164: #endif
3165:
3166: /*
3167: * Find out how much physical memory we have and in how many chunks.
3168: */
3169: for (mem_cnt = 0, mp = mem; mp->size; mp++) {
3170: if (mp->start >= pmap_memlimit)
3171: continue;
3172: if (mp->start + mp->size > pmap_memlimit) {
3173: size = pmap_memlimit - mp->start;
3174: physmem += btoc(size);
3175: } else {
3176: physmem += btoc(mp->size);
3177: }
3178: mem_cnt++;
3179: }
3180:
3181: /*
3182: * Count the number of available entries.
3183: */
3184: for (avail_cnt = 0, mp = avail; mp->size; mp++)
3185: avail_cnt++;
3186:
3187: /*
3188: * Page align all regions.
3189: */
3190: kernelstart = trunc_page(kernelstart);
3191: kernelend = round_page(kernelend);
3192: for (mp = avail, i = 0; i < avail_cnt; i++, mp++) {
3193: s = round_page(mp->start);
3194: mp->size -= (s - mp->start);
3195: mp->size = trunc_page(mp->size);
3196: mp->start = s;
3197: e = mp->start + mp->size;
3198:
3199: DPRINTFN(BOOT,
1.85 matt 3200: "pmap_bootstrap: b-avail[%d] start %#" _PRIxpa " size %#" _PRIxpa "\n",
3201: i, mp->start, mp->size);
1.1 matt 3202:
3203: /*
3204: * Don't allow the end to run beyond our artificial limit
3205: */
3206: if (e > pmap_memlimit)
3207: e = pmap_memlimit;
3208:
3209: /*
3210: * Is this region empty or strange? skip it.
3211: */
3212: if (e <= s) {
3213: mp->start = 0;
3214: mp->size = 0;
3215: continue;
3216: }
3217:
3218: /*
3219: * Does this overlap the beginning of kernel?
3220: * Does extend past the end of the kernel?
3221: */
3222: else if (s < kernelstart && e > kernelstart) {
3223: if (e > kernelend) {
3224: avail[avail_cnt].start = kernelend;
3225: avail[avail_cnt].size = e - kernelend;
3226: avail_cnt++;
3227: }
3228: mp->size = kernelstart - s;
3229: }
3230: /*
3231: * Check whether this region overlaps the end of the kernel.
3232: */
3233: else if (s < kernelend && e > kernelend) {
3234: mp->start = kernelend;
3235: mp->size = e - kernelend;
3236: }
3237: /*
3238: * Look whether this regions is completely inside the kernel.
3239: * Nuke it if it does.
3240: */
3241: else if (s >= kernelstart && e <= kernelend) {
3242: mp->start = 0;
3243: mp->size = 0;
3244: }
3245: /*
3246: * If the user imposed a memory limit, enforce it.
3247: */
3248: else if (s >= pmap_memlimit) {
1.6 thorpej 3249: mp->start = -PAGE_SIZE; /* let's know why */
1.1 matt 3250: mp->size = 0;
3251: }
3252: else {
3253: mp->start = s;
3254: mp->size = e - s;
3255: }
3256: DPRINTFN(BOOT,
1.85 matt 3257: "pmap_bootstrap: a-avail[%d] start %#" _PRIxpa " size %#" _PRIxpa "\n",
3258: i, mp->start, mp->size);
1.1 matt 3259: }
3260:
3261: /*
3262: * Move (and uncount) all the null return to the end.
3263: */
3264: for (mp = avail, i = 0; i < avail_cnt; i++, mp++) {
3265: if (mp->size == 0) {
3266: tmp = avail[i];
3267: avail[i] = avail[--avail_cnt];
3268: avail[avail_cnt] = avail[i];
3269: }
3270: }
3271:
3272: /*
1.61 skrll 3273: * (Bubble)sort them into ascending order.
1.1 matt 3274: */
3275: for (i = 0; i < avail_cnt; i++) {
3276: for (j = i + 1; j < avail_cnt; j++) {
3277: if (avail[i].start > avail[j].start) {
3278: tmp = avail[i];
3279: avail[i] = avail[j];
3280: avail[j] = tmp;
3281: }
3282: }
3283: }
3284:
3285: /*
3286: * Make sure they don't overlap.
3287: */
3288: for (mp = avail, i = 0; i < avail_cnt - 1; i++, mp++) {
3289: if (mp[0].start + mp[0].size > mp[1].start) {
3290: mp[0].size = mp[1].start - mp[0].start;
3291: }
3292: DPRINTFN(BOOT,
1.85 matt 3293: "pmap_bootstrap: avail[%d] start %#" _PRIxpa " size %#" _PRIxpa "\n",
3294: i, mp->start, mp->size);
1.1 matt 3295: }
3296: DPRINTFN(BOOT,
1.85 matt 3297: "pmap_bootstrap: avail[%d] start %#" _PRIxpa " size %#" _PRIxpa "\n",
3298: i, mp->start, mp->size);
1.1 matt 3299:
3300: #ifdef PTEGCOUNT
3301: pmap_pteg_cnt = PTEGCOUNT;
3302: #else /* PTEGCOUNT */
1.38 sanjayl 3303:
1.1 matt 3304: pmap_pteg_cnt = 0x1000;
3305:
3306: while (pmap_pteg_cnt < physmem)
3307: pmap_pteg_cnt <<= 1;
3308:
3309: pmap_pteg_cnt >>= 1;
3310: #endif /* PTEGCOUNT */
3311:
1.38 sanjayl 3312: #ifdef DEBUG
1.85 matt 3313: DPRINTFN(BOOT, "pmap_pteg_cnt: 0x%x\n", pmap_pteg_cnt);
1.38 sanjayl 3314: #endif
3315:
1.1 matt 3316: /*
3317: * Find suitably aligned memory for PTEG hash table.
3318: */
1.2 matt 3319: size = pmap_pteg_cnt * sizeof(struct pteg);
1.53 garbled 3320: pmap_pteg_table = (void *)(uintptr_t) pmap_boot_find_memory(size, size, 0);
1.38 sanjayl 3321:
3322: #ifdef DEBUG
3323: DPRINTFN(BOOT,
1.85 matt 3324: "PTEG cnt: 0x%x HTAB size: 0x%08x bytes, address: %p\n", pmap_pteg_cnt, (unsigned int)size, pmap_pteg_table);
1.38 sanjayl 3325: #endif
3326:
3327:
1.1 matt 3328: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
3329: if ( (uintptr_t) pmap_pteg_table + size > SEGMENT_LENGTH)
1.54 mlelstv 3330: panic("pmap_bootstrap: pmap_pteg_table end (%p + %#" _PRIxpa ") > 256MB",
1.1 matt 3331: pmap_pteg_table, size);
3332: #endif
3333:
1.32 he 3334: memset(__UNVOLATILE(pmap_pteg_table), 0,
3335: pmap_pteg_cnt * sizeof(struct pteg));
1.1 matt 3336: pmap_pteg_mask = pmap_pteg_cnt - 1;
3337:
3338: /*
3339: * We cannot do pmap_steal_memory here since UVM hasn't been loaded
3340: * with pages. So we just steal them before giving them to UVM.
3341: */
3342: size = sizeof(pmap_pvo_table[0]) * pmap_pteg_cnt;
1.53 garbled 3343: pmap_pvo_table = (void *)(uintptr_t) pmap_boot_find_memory(size, PAGE_SIZE, 0);
1.1 matt 3344: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
3345: if ( (uintptr_t) pmap_pvo_table + size > SEGMENT_LENGTH)
1.54 mlelstv 3346: panic("pmap_bootstrap: pmap_pvo_table end (%p + %#" _PRIxpa ") > 256MB",
1.1 matt 3347: pmap_pvo_table, size);
3348: #endif
3349:
3350: for (i = 0; i < pmap_pteg_cnt; i++)
3351: TAILQ_INIT(&pmap_pvo_table[i]);
3352:
3353: #ifndef MSGBUFADDR
3354: /*
3355: * Allocate msgbuf in high memory.
3356: */
1.53 garbled 3357: msgbuf_paddr = pmap_boot_find_memory(MSGBUFSIZE, PAGE_SIZE, 1);
1.1 matt 3358: #endif
3359:
3360: for (mp = avail, i = 0; i < avail_cnt; mp++, i++) {
3361: paddr_t pfstart = atop(mp->start);
3362: paddr_t pfend = atop(mp->start + mp->size);
3363: if (mp->size == 0)
3364: continue;
3365: if (mp->start + mp->size <= SEGMENT_LENGTH) {
3366: uvm_page_physload(pfstart, pfend, pfstart, pfend,
3367: VM_FREELIST_FIRST256);
3368: } else if (mp->start >= SEGMENT_LENGTH) {
3369: uvm_page_physload(pfstart, pfend, pfstart, pfend,
3370: VM_FREELIST_DEFAULT);
3371: } else {
3372: pfend = atop(SEGMENT_LENGTH);
3373: uvm_page_physload(pfstart, pfend, pfstart, pfend,
3374: VM_FREELIST_FIRST256);
3375: pfstart = atop(SEGMENT_LENGTH);
3376: pfend = atop(mp->start + mp->size);
3377: uvm_page_physload(pfstart, pfend, pfstart, pfend,
3378: VM_FREELIST_DEFAULT);
3379: }
3380: }
3381:
3382: /*
3383: * Make sure kernel vsid is allocated as well as VSID 0.
3384: */
3385: pmap_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS-1)) / VSID_NBPW]
3386: |= 1 << (KERNEL_VSIDBITS % VSID_NBPW);
1.53 garbled 3387: pmap_vsid_bitmap[(PHYSMAP_VSIDBITS & (NPMAPS-1)) / VSID_NBPW]
3388: |= 1 << (PHYSMAP_VSIDBITS % VSID_NBPW);
1.1 matt 3389: pmap_vsid_bitmap[0] |= 1;
3390:
3391: /*
1.103 thorpej 3392: * Initialize kernel pmap.
1.1 matt 3393: */
1.103 thorpej 3394: #if defined(PMAP_OEA) || defined(PMAP_OEA64_BRIDGE)
1.1 matt 3395: for (i = 0; i < 16; i++) {
1.38 sanjayl 3396: pmap_kernel()->pm_sr[i] = KERNELN_SEGMENT(i)|SR_PRKEY;
1.1 matt 3397: }
1.102 thorpej 3398: pmap_kernel()->pm_vsid = KERNEL_VSIDBITS;
1.1 matt 3399:
3400: pmap_kernel()->pm_sr[KERNEL_SR] = KERNEL_SEGMENT|SR_SUKEY|SR_PRKEY;
3401: #ifdef KERNEL2_SR
3402: pmap_kernel()->pm_sr[KERNEL2_SR] = KERNEL2_SEGMENT|SR_SUKEY|SR_PRKEY;
3403: #endif
1.53 garbled 3404: #endif /* PMAP_OEA || PMAP_OEA64_BRIDGE */
1.103 thorpej 3405:
3406: #if defined(PMAP_OEA) && defined(PPC_OEA601)
1.105 thorpej 3407: if ((MFPVR() >> 16) == MPC601) {
1.103 thorpej 3408: for (i = 0; i < 16; i++) {
3409: if (iosrtable[i] & SR601_T) {
3410: pmap_kernel()->pm_sr[i] = iosrtable[i];
3411: }
1.1 matt 3412: }
3413: }
1.103 thorpej 3414: #endif /* PMAP_OEA && PPC_OEA601 */
1.1 matt 3415:
3416: #ifdef ALTIVEC
3417: pmap_use_altivec = cpu_altivec;
3418: #endif
3419:
3420: #ifdef DEBUG
3421: if (pmapdebug & PMAPDEBUG_BOOT) {
3422: u_int cnt;
1.94 cherry 3423: uvm_physseg_t bank;
1.1 matt 3424: char pbuf[9];
1.94 cherry 3425: for (cnt = 0, bank = uvm_physseg_get_first();
3426: uvm_physseg_valid_p(bank);
3427: bank = uvm_physseg_get_next(bank)) {
3428: cnt += uvm_physseg_get_avail_end(bank) -
3429: uvm_physseg_get_avail_start(bank);
1.53 garbled 3430: printf("pmap_bootstrap: vm_physmem[%d]=%#" _PRIxpa "-%#" _PRIxpa "/%#" _PRIxpa "\n",
1.1 matt 3431: bank,
1.94 cherry 3432: ptoa(uvm_physseg_get_avail_start(bank)),
3433: ptoa(uvm_physseg_get_avail_end(bank)),
3434: ptoa(uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank)));
1.1 matt 3435: }
3436: format_bytes(pbuf, sizeof(pbuf), ptoa((u_int64_t) cnt));
3437: printf("pmap_bootstrap: UVM memory = %s (%u pages)\n",
3438: pbuf, cnt);
3439: }
3440: #endif
3441:
1.106 martin 3442: pool_init(&pmap_pvo_pool, sizeof(struct pvo_entry),
3443: sizeof(struct pvo_entry), 0, 0, "pmap_pvopl",
3444: &pmap_pool_allocator, IPL_VM);
1.1 matt 3445:
1.106 martin 3446: pool_setlowat(&pmap_pvo_pool, 1008);
1.1 matt 3447:
3448: pool_init(&pmap_pool, sizeof(struct pmap),
1.106 martin 3449: sizeof(void *), 0, 0, "pmap_pl", &pmap_pool_allocator,
1.48 ad 3450: IPL_NONE);
1.41 matt 3451:
1.89 macallan 3452: #if defined(PMAP_NEED_MAPKERNEL)
1.41 matt 3453: {
1.53 garbled 3454: struct pmap *pm = pmap_kernel();
1.58 garbled 3455: #if defined(PMAP_NEED_FULL_MAPKERNEL)
1.41 matt 3456: extern int etext[], kernel_text[];
3457: vaddr_t va, va_etext = (paddr_t) etext;
1.53 garbled 3458: #endif
3459: paddr_t pa, pa_end;
1.42 matt 3460: register_t sr;
1.53 garbled 3461: struct pte pt;
3462: unsigned int ptegidx;
3463: int bank;
1.42 matt 3464:
1.53 garbled 3465: sr = PHYSMAPN_SEGMENT(0) | SR_SUKEY|SR_PRKEY;
3466: pm->pm_sr[0] = sr;
1.41 matt 3467:
1.53 garbled 3468: for (bank = 0; bank < vm_nphysseg; bank++) {
1.73 uebayasi 3469: pa_end = ptoa(VM_PHYSMEM_PTR(bank)->avail_end);
3470: pa = ptoa(VM_PHYSMEM_PTR(bank)->avail_start);
1.53 garbled 3471: for (; pa < pa_end; pa += PAGE_SIZE) {
3472: ptegidx = va_to_pteg(pm, pa);
3473: pmap_pte_create(&pt, pm, pa, pa | PTE_M|PTE_BW);
3474: pmap_pte_insert(ptegidx, &pt);
3475: }
3476: }
3477:
1.58 garbled 3478: #if defined(PMAP_NEED_FULL_MAPKERNEL)
1.41 matt 3479: va = (vaddr_t) kernel_text;
3480:
3481: for (pa = kernelstart; va < va_etext;
1.53 garbled 3482: pa += PAGE_SIZE, va += PAGE_SIZE) {
3483: ptegidx = va_to_pteg(pm, va);
3484: pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BR);
3485: pmap_pte_insert(ptegidx, &pt);
3486: }
1.41 matt 3487:
3488: for (; pa < kernelend;
1.53 garbled 3489: pa += PAGE_SIZE, va += PAGE_SIZE) {
3490: ptegidx = va_to_pteg(pm, va);
3491: pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BW);
3492: pmap_pte_insert(ptegidx, &pt);
3493: }
3494:
1.58 garbled 3495: for (va = 0, pa = 0; va < kernelstart;
1.53 garbled 3496: pa += PAGE_SIZE, va += PAGE_SIZE) {
3497: ptegidx = va_to_pteg(pm, va);
1.58 garbled 3498: if (va < 0x3000)
3499: pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BR);
3500: else
3501: pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BW);
3502: pmap_pte_insert(ptegidx, &pt);
3503: }
3504: for (va = kernelend, pa = kernelend; va < SEGMENT_LENGTH;
3505: pa += PAGE_SIZE, va += PAGE_SIZE) {
3506: ptegidx = va_to_pteg(pm, va);
1.53 garbled 3507: pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BW);
3508: pmap_pte_insert(ptegidx, &pt);
3509: }
1.103 thorpej 3510: #endif /* PMAP_NEED_FULL_MAPKERNEL */
1.99 thorpej 3511: }
1.103 thorpej 3512: #endif /* PMAP_NEED_MAPKERNEL */
1.99 thorpej 3513: }
1.42 matt 3514:
1.99 thorpej 3515: /*
3516: * Using the data structures prepared in pmap_bootstrap1(), program
3517: * the MMU hardware.
3518: */
3519: void
3520: pmap_bootstrap2(void)
3521: {
1.103 thorpej 3522: #if defined(PMAP_OEA) || defined(PMAP_OEA64_BRIDGE)
1.99 thorpej 3523: for (int i = 0; i < 16; i++) {
3524: __asm volatile("mtsrin %0,%1"
3525: :: "r"(pmap_kernel()->pm_sr[i]),
3526: "r"(i << ADDR_SR_SHFT));
1.41 matt 3527: }
1.99 thorpej 3528: #endif /* PMAP_OEA || PMAP_OEA64_BRIDGE */
1.103 thorpej 3529:
3530: #if defined(PMAP_OEA)
1.109 riastrad 3531: __asm volatile("sync; mtsdr1 %0; isync"
3532: :
3533: : "r"((uintptr_t)pmap_pteg_table | (pmap_pteg_mask >> 10))
3534: : "memory");
1.103 thorpej 3535: #elif defined(PMAP_OEA64) || defined(PMAP_OEA64_BRIDGE)
1.109 riastrad 3536: __asm volatile("sync; mtsdr1 %0; isync"
3537: :
3538: : "r"((uintptr_t)pmap_pteg_table |
3539: (32 - __builtin_clz(pmap_pteg_mask >> 11)))
3540: : "memory");
1.41 matt 3541: #endif
1.99 thorpej 3542: tlbia();
1.91 macallan 3543:
3544: #if defined(PMAPDEBUG)
1.103 thorpej 3545: if (pmapdebug)
1.91 macallan 3546: pmap_print_mmuregs();
3547: #endif
1.1 matt 3548: }
1.99 thorpej 3549:
3550: /*
3551: * This is not part of the defined PMAP interface and is specific to the
3552: * PowerPC architecture. This is called during initppc, before the system
3553: * is really initialized.
3554: */
3555: void
3556: pmap_bootstrap(paddr_t kernelstart, paddr_t kernelend)
3557: {
3558: pmap_bootstrap1(kernelstart, kernelend);
3559: pmap_bootstrap2();
3560: }
CVSweb <webmaster@jp.NetBSD.org>