Annotation of src/sys/arch/powerpc/oea/pmap.c, Revision 1.99
1.99 ! thorpej 1: /* $NetBSD: pmap.c,v 1.98 2020/07/06 09:34:17 rin Exp $ */
1.1 matt 2: /*-
3: * Copyright (c) 2001 The NetBSD Foundation, Inc.
4: * All rights reserved.
5: *
6: * This code is derived from software contributed to The NetBSD Foundation
7: * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc.
8: *
1.38 sanjayl 9: * Support for PPC64 Bridge mode added by Sanjay Lal <sanjayl@kymasys.com>
10: * of Kyma Systems LLC.
11: *
1.1 matt 12: * Redistribution and use in source and binary forms, with or without
13: * modification, are permitted provided that the following conditions
14: * are met:
15: * 1. Redistributions of source code must retain the above copyright
16: * notice, this list of conditions and the following disclaimer.
17: * 2. Redistributions in binary form must reproduce the above copyright
18: * notice, this list of conditions and the following disclaimer in the
19: * documentation and/or other materials provided with the distribution.
20: *
21: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
22: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
25: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31: * POSSIBILITY OF SUCH DAMAGE.
32: */
33:
34: /*
35: * Copyright (C) 1995, 1996 Wolfgang Solfrank.
36: * Copyright (C) 1995, 1996 TooLs GmbH.
37: * All rights reserved.
38: *
39: * Redistribution and use in source and binary forms, with or without
40: * modification, are permitted provided that the following conditions
41: * are met:
42: * 1. Redistributions of source code must retain the above copyright
43: * notice, this list of conditions and the following disclaimer.
44: * 2. Redistributions in binary form must reproduce the above copyright
45: * notice, this list of conditions and the following disclaimer in the
46: * documentation and/or other materials provided with the distribution.
47: * 3. All advertising materials mentioning features or use of this software
48: * must display the following acknowledgement:
49: * This product includes software developed by TooLs GmbH.
50: * 4. The name of TooLs GmbH may not be used to endorse or promote products
51: * derived from this software without specific prior written permission.
52: *
53: * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
54: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
55: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
56: * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
57: * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
58: * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
59: * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
60: * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
61: * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
62: * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
63: */
1.11 lukem 64:
65: #include <sys/cdefs.h>
1.99 ! thorpej 66: __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.98 2020/07/06 09:34:17 rin Exp $");
1.53 garbled 67:
68: #define PMAP_NOOPNAMES
1.1 matt 69:
1.98 rin 70: #ifdef _KERNEL_OPT
1.1 matt 71: #include "opt_altivec.h"
1.57 matt 72: #include "opt_multiprocessor.h"
1.1 matt 73: #include "opt_pmap.h"
1.98 rin 74: #include "opt_ppcarch.h"
75: #endif
1.57 matt 76:
1.1 matt 77: #include <sys/param.h>
78: #include <sys/proc.h>
79: #include <sys/pool.h>
80: #include <sys/queue.h>
81: #include <sys/device.h> /* for evcnt */
82: #include <sys/systm.h>
1.50 ad 83: #include <sys/atomic.h>
1.1 matt 84:
85: #include <uvm/uvm.h>
1.94 cherry 86: #include <uvm/uvm_physseg.h>
1.1 matt 87:
88: #include <machine/powerpc.h>
1.80 matt 89: #include <powerpc/bat.h>
90: #include <powerpc/pcb.h>
91: #include <powerpc/psl.h>
1.1 matt 92: #include <powerpc/spr.h>
1.71 matt 93: #include <powerpc/oea/spr.h>
94: #include <powerpc/oea/sr_601.h>
1.1 matt 95:
96: #ifdef ALTIVEC
1.86 matt 97: extern int pmap_use_altivec;
1.1 matt 98: #endif
99:
1.21 aymeric 100: #ifdef PMAP_MEMLIMIT
1.53 garbled 101: static paddr_t pmap_memlimit = PMAP_MEMLIMIT;
1.21 aymeric 102: #else
1.53 garbled 103: static paddr_t pmap_memlimit = -PAGE_SIZE; /* there is no limit */
1.21 aymeric 104: #endif
1.1 matt 105:
1.86 matt 106: extern struct pmap kernel_pmap_;
107: static unsigned int pmap_pages_stolen;
108: static u_long pmap_pte_valid;
1.1 matt 109: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1.86 matt 110: static u_long pmap_pvo_enter_depth;
111: static u_long pmap_pvo_remove_depth;
1.1 matt 112: #endif
113:
114: #ifndef MSGBUFADDR
115: extern paddr_t msgbuf_paddr;
116: #endif
117:
118: static struct mem_region *mem, *avail;
119: static u_int mem_cnt, avail_cnt;
120:
1.53 garbled 121: #if !defined(PMAP_OEA64) && !defined(PMAP_OEA64_BRIDGE)
122: # define PMAP_OEA 1
123: #endif
124:
125: #if defined(PMAP_OEA)
126: #define _PRIxpte "lx"
127: #else
128: #define _PRIxpte PRIx64
129: #endif
130: #define _PRIxpa "lx"
131: #define _PRIxva "lx"
1.54 mlelstv 132: #define _PRIsr "lx"
1.53 garbled 133:
1.76 matt 134: #ifdef PMAP_NEEDS_FIXUP
1.53 garbled 135: #if defined(PMAP_OEA)
136: #define PMAPNAME(name) pmap32_##name
137: #elif defined(PMAP_OEA64)
138: #define PMAPNAME(name) pmap64_##name
139: #elif defined(PMAP_OEA64_BRIDGE)
140: #define PMAPNAME(name) pmap64bridge_##name
141: #else
142: #error unknown variant for pmap
143: #endif
1.76 matt 144: #endif /* PMAP_NEEDS_FIXUP */
1.53 garbled 145:
1.76 matt 146: #ifdef PMAPNAME
1.53 garbled 147: #define STATIC static
148: #define pmap_pte_spill PMAPNAME(pte_spill)
149: #define pmap_real_memory PMAPNAME(real_memory)
150: #define pmap_init PMAPNAME(init)
151: #define pmap_virtual_space PMAPNAME(virtual_space)
152: #define pmap_create PMAPNAME(create)
153: #define pmap_reference PMAPNAME(reference)
154: #define pmap_destroy PMAPNAME(destroy)
155: #define pmap_copy PMAPNAME(copy)
156: #define pmap_update PMAPNAME(update)
157: #define pmap_enter PMAPNAME(enter)
158: #define pmap_remove PMAPNAME(remove)
159: #define pmap_kenter_pa PMAPNAME(kenter_pa)
160: #define pmap_kremove PMAPNAME(kremove)
161: #define pmap_extract PMAPNAME(extract)
162: #define pmap_protect PMAPNAME(protect)
163: #define pmap_unwire PMAPNAME(unwire)
164: #define pmap_page_protect PMAPNAME(page_protect)
165: #define pmap_query_bit PMAPNAME(query_bit)
166: #define pmap_clear_bit PMAPNAME(clear_bit)
167:
168: #define pmap_activate PMAPNAME(activate)
169: #define pmap_deactivate PMAPNAME(deactivate)
170:
171: #define pmap_pinit PMAPNAME(pinit)
172: #define pmap_procwr PMAPNAME(procwr)
173:
1.86 matt 174: #define pmap_pool PMAPNAME(pool)
175: #define pmap_upvo_pool PMAPNAME(upvo_pool)
176: #define pmap_mpvo_pool PMAPNAME(mpvo_pool)
177: #define pmap_pvo_table PMAPNAME(pvo_table)
1.53 garbled 178: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
179: #define pmap_pte_print PMAPNAME(pte_print)
180: #define pmap_pteg_check PMAPNAME(pteg_check)
181: #define pmap_print_mmruregs PMAPNAME(print_mmuregs)
182: #define pmap_print_pte PMAPNAME(print_pte)
183: #define pmap_pteg_dist PMAPNAME(pteg_dist)
184: #endif
185: #if defined(DEBUG) || defined(PMAPCHECK)
186: #define pmap_pvo_verify PMAPNAME(pvo_verify)
1.56 phx 187: #define pmapcheck PMAPNAME(check)
188: #endif
189: #if defined(DEBUG) || defined(PMAPDEBUG)
190: #define pmapdebug PMAPNAME(debug)
1.53 garbled 191: #endif
192: #define pmap_steal_memory PMAPNAME(steal_memory)
193: #define pmap_bootstrap PMAPNAME(bootstrap)
194: #else
195: #define STATIC /* nothing */
196: #endif /* PMAPNAME */
197:
198: STATIC int pmap_pte_spill(struct pmap *, vaddr_t, bool);
199: STATIC void pmap_real_memory(paddr_t *, psize_t *);
200: STATIC void pmap_init(void);
201: STATIC void pmap_virtual_space(vaddr_t *, vaddr_t *);
202: STATIC pmap_t pmap_create(void);
203: STATIC void pmap_reference(pmap_t);
204: STATIC void pmap_destroy(pmap_t);
205: STATIC void pmap_copy(pmap_t, pmap_t, vaddr_t, vsize_t, vaddr_t);
206: STATIC void pmap_update(pmap_t);
1.65 cegger 207: STATIC int pmap_enter(pmap_t, vaddr_t, paddr_t, vm_prot_t, u_int);
1.53 garbled 208: STATIC void pmap_remove(pmap_t, vaddr_t, vaddr_t);
1.68 cegger 209: STATIC void pmap_kenter_pa(vaddr_t, paddr_t, vm_prot_t, u_int);
1.53 garbled 210: STATIC void pmap_kremove(vaddr_t, vsize_t);
211: STATIC bool pmap_extract(pmap_t, vaddr_t, paddr_t *);
212:
213: STATIC void pmap_protect(pmap_t, vaddr_t, vaddr_t, vm_prot_t);
214: STATIC void pmap_unwire(pmap_t, vaddr_t);
215: STATIC void pmap_page_protect(struct vm_page *, vm_prot_t);
216: STATIC bool pmap_query_bit(struct vm_page *, int);
217: STATIC bool pmap_clear_bit(struct vm_page *, int);
218:
219: STATIC void pmap_activate(struct lwp *);
220: STATIC void pmap_deactivate(struct lwp *);
221:
222: STATIC void pmap_pinit(pmap_t pm);
223: STATIC void pmap_procwr(struct proc *, vaddr_t, size_t);
224:
225: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
226: STATIC void pmap_pte_print(volatile struct pte *);
227: STATIC void pmap_pteg_check(void);
228: STATIC void pmap_print_mmuregs(void);
229: STATIC void pmap_print_pte(pmap_t, vaddr_t);
230: STATIC void pmap_pteg_dist(void);
231: #endif
232: #if defined(DEBUG) || defined(PMAPCHECK)
233: STATIC void pmap_pvo_verify(void);
234: #endif
235: STATIC vaddr_t pmap_steal_memory(vsize_t, vaddr_t *, vaddr_t *);
236: STATIC void pmap_bootstrap(paddr_t, paddr_t);
237:
238: #ifdef PMAPNAME
239: const struct pmap_ops PMAPNAME(ops) = {
240: .pmapop_pte_spill = pmap_pte_spill,
241: .pmapop_real_memory = pmap_real_memory,
242: .pmapop_init = pmap_init,
243: .pmapop_virtual_space = pmap_virtual_space,
244: .pmapop_create = pmap_create,
245: .pmapop_reference = pmap_reference,
246: .pmapop_destroy = pmap_destroy,
247: .pmapop_copy = pmap_copy,
248: .pmapop_update = pmap_update,
249: .pmapop_enter = pmap_enter,
250: .pmapop_remove = pmap_remove,
251: .pmapop_kenter_pa = pmap_kenter_pa,
252: .pmapop_kremove = pmap_kremove,
253: .pmapop_extract = pmap_extract,
254: .pmapop_protect = pmap_protect,
255: .pmapop_unwire = pmap_unwire,
256: .pmapop_page_protect = pmap_page_protect,
257: .pmapop_query_bit = pmap_query_bit,
258: .pmapop_clear_bit = pmap_clear_bit,
259: .pmapop_activate = pmap_activate,
260: .pmapop_deactivate = pmap_deactivate,
261: .pmapop_pinit = pmap_pinit,
262: .pmapop_procwr = pmap_procwr,
263: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
264: .pmapop_pte_print = pmap_pte_print,
265: .pmapop_pteg_check = pmap_pteg_check,
266: .pmapop_print_mmuregs = pmap_print_mmuregs,
267: .pmapop_print_pte = pmap_print_pte,
268: .pmapop_pteg_dist = pmap_pteg_dist,
269: #else
270: .pmapop_pte_print = NULL,
271: .pmapop_pteg_check = NULL,
272: .pmapop_print_mmuregs = NULL,
273: .pmapop_print_pte = NULL,
274: .pmapop_pteg_dist = NULL,
275: #endif
276: #if defined(DEBUG) || defined(PMAPCHECK)
277: .pmapop_pvo_verify = pmap_pvo_verify,
278: #else
279: .pmapop_pvo_verify = NULL,
1.1 matt 280: #endif
1.53 garbled 281: .pmapop_steal_memory = pmap_steal_memory,
282: .pmapop_bootstrap = pmap_bootstrap,
283: };
284: #endif /* !PMAPNAME */
1.1 matt 285:
286: /*
1.38 sanjayl 287: * The following structure is aligned to 32 bytes
1.1 matt 288: */
289: struct pvo_entry {
290: LIST_ENTRY(pvo_entry) pvo_vlink; /* Link to common virt page */
291: TAILQ_ENTRY(pvo_entry) pvo_olink; /* Link to overflow entry */
292: struct pte pvo_pte; /* Prebuilt PTE */
293: pmap_t pvo_pmap; /* ptr to owning pmap */
294: vaddr_t pvo_vaddr; /* VA of entry */
295: #define PVO_PTEGIDX_MASK 0x0007 /* which PTEG slot */
296: #define PVO_PTEGIDX_VALID 0x0008 /* slot is valid */
297: #define PVO_WIRED 0x0010 /* PVO entry is wired */
298: #define PVO_MANAGED 0x0020 /* PVO e. for managed page */
299: #define PVO_EXECUTABLE 0x0040 /* PVO e. for executable page */
1.39 matt 300: #define PVO_WIRED_P(pvo) ((pvo)->pvo_vaddr & PVO_WIRED)
301: #define PVO_MANAGED_P(pvo) ((pvo)->pvo_vaddr & PVO_MANAGED)
302: #define PVO_EXECUTABLE_P(pvo) ((pvo)->pvo_vaddr & PVO_EXECUTABLE)
1.12 matt 303: #define PVO_ENTER_INSERT 0 /* PVO has been removed */
304: #define PVO_SPILL_UNSET 1 /* PVO has been evicted */
305: #define PVO_SPILL_SET 2 /* PVO has been spilled */
306: #define PVO_SPILL_INSERT 3 /* PVO has been inserted */
307: #define PVO_PMAP_PAGE_PROTECT 4 /* PVO has changed */
308: #define PVO_PMAP_PROTECT 5 /* PVO has changed */
309: #define PVO_REMOVE 6 /* PVO has been removed */
310: #define PVO_WHERE_MASK 15
311: #define PVO_WHERE_SHFT 8
1.38 sanjayl 312: } __attribute__ ((aligned (32)));
1.1 matt 313: #define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF)
314: #define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
315: #define PVO_PTEGIDX_ISSET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID)
316: #define PVO_PTEGIDX_CLR(pvo) \
317: ((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK)))
318: #define PVO_PTEGIDX_SET(pvo,i) \
319: ((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID))
1.12 matt 320: #define PVO_WHERE(pvo,w) \
321: ((pvo)->pvo_vaddr &= ~(PVO_WHERE_MASK << PVO_WHERE_SHFT), \
322: (pvo)->pvo_vaddr |= ((PVO_ ## w) << PVO_WHERE_SHFT))
1.1 matt 323:
324: TAILQ_HEAD(pvo_tqhead, pvo_entry);
325: struct pvo_tqhead *pmap_pvo_table; /* pvo entries by ptegroup index */
1.53 garbled 326: static struct pvo_head pmap_pvo_kunmanaged = LIST_HEAD_INITIALIZER(pmap_pvo_kunmanaged); /* list of unmanaged pages */
327: static struct pvo_head pmap_pvo_unmanaged = LIST_HEAD_INITIALIZER(pmap_pvo_unmanaged); /* list of unmanaged pages */
1.1 matt 328:
329: struct pool pmap_pool; /* pool for pmap structures */
330: struct pool pmap_upvo_pool; /* pool for pvo entries for unmanaged pages */
331: struct pool pmap_mpvo_pool; /* pool for pvo entries for managed pages */
332:
333: /*
334: * We keep a cache of unmanaged pages to be used for pvo entries for
335: * unmanaged pages.
336: */
337: struct pvo_page {
338: SIMPLEQ_ENTRY(pvo_page) pvop_link;
339: };
340: SIMPLEQ_HEAD(pvop_head, pvo_page);
1.53 garbled 341: static struct pvop_head pmap_upvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_upvop_head);
342: static struct pvop_head pmap_mpvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_mpvop_head);
1.86 matt 343: static u_long pmap_upvop_free;
344: static u_long pmap_upvop_maxfree;
345: static u_long pmap_mpvop_free;
346: static u_long pmap_mpvop_maxfree;
1.1 matt 347:
1.53 garbled 348: static void *pmap_pool_ualloc(struct pool *, int);
349: static void *pmap_pool_malloc(struct pool *, int);
1.1 matt 350:
1.53 garbled 351: static void pmap_pool_ufree(struct pool *, void *);
352: static void pmap_pool_mfree(struct pool *, void *);
1.1 matt 353:
354: static struct pool_allocator pmap_pool_mallocator = {
1.43 garbled 355: .pa_alloc = pmap_pool_malloc,
356: .pa_free = pmap_pool_mfree,
357: .pa_pagesz = 0,
1.1 matt 358: };
359:
360: static struct pool_allocator pmap_pool_uallocator = {
1.43 garbled 361: .pa_alloc = pmap_pool_ualloc,
362: .pa_free = pmap_pool_ufree,
363: .pa_pagesz = 0,
1.1 matt 364: };
365:
366: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
1.2 matt 367: void pmap_pte_print(volatile struct pte *);
1.1 matt 368: void pmap_pteg_check(void);
369: void pmap_pteg_dist(void);
370: void pmap_print_pte(pmap_t, vaddr_t);
371: void pmap_print_mmuregs(void);
372: #endif
373:
374: #if defined(DEBUG) || defined(PMAPCHECK)
375: #ifdef PMAPCHECK
376: int pmapcheck = 1;
377: #else
378: int pmapcheck = 0;
379: #endif
380: void pmap_pvo_verify(void);
1.53 garbled 381: static void pmap_pvo_check(const struct pvo_entry *);
1.1 matt 382: #define PMAP_PVO_CHECK(pvo) \
383: do { \
384: if (pmapcheck) \
385: pmap_pvo_check(pvo); \
386: } while (0)
387: #else
388: #define PMAP_PVO_CHECK(pvo) do { } while (/*CONSTCOND*/0)
389: #endif
1.53 garbled 390: static int pmap_pte_insert(int, struct pte *);
391: static int pmap_pvo_enter(pmap_t, struct pool *, struct pvo_head *,
1.2 matt 392: vaddr_t, paddr_t, register_t, int);
1.53 garbled 393: static void pmap_pvo_remove(struct pvo_entry *, int, struct pvo_head *);
394: static void pmap_pvo_free(struct pvo_entry *);
395: static void pmap_pvo_free_list(struct pvo_head *);
396: static struct pvo_entry *pmap_pvo_find_va(pmap_t, vaddr_t, int *);
397: static volatile struct pte *pmap_pvo_to_pte(const struct pvo_entry *, int);
398: static struct pvo_entry *pmap_pvo_reclaim(struct pmap *);
399: static void pvo_set_exec(struct pvo_entry *);
400: static void pvo_clear_exec(struct pvo_entry *);
1.1 matt 401:
1.53 garbled 402: static void tlbia(void);
1.1 matt 403:
1.53 garbled 404: static void pmap_release(pmap_t);
405: static paddr_t pmap_boot_find_memory(psize_t, psize_t, int);
1.1 matt 406:
1.25 chs 407: static uint32_t pmap_pvo_reclaim_nextidx;
408: #ifdef DEBUG
409: static int pmap_pvo_reclaim_debugctr;
410: #endif
411:
1.1 matt 412: #define VSID_NBPW (sizeof(uint32_t) * 8)
413: static uint32_t pmap_vsid_bitmap[NPMAPS / VSID_NBPW];
414:
415: static int pmap_initialized;
416:
417: #if defined(DEBUG) || defined(PMAPDEBUG)
418: #define PMAPDEBUG_BOOT 0x0001
419: #define PMAPDEBUG_PTE 0x0002
420: #define PMAPDEBUG_EXEC 0x0008
421: #define PMAPDEBUG_PVOENTER 0x0010
422: #define PMAPDEBUG_PVOREMOVE 0x0020
423: #define PMAPDEBUG_ACTIVATE 0x0100
424: #define PMAPDEBUG_CREATE 0x0200
425: #define PMAPDEBUG_ENTER 0x1000
426: #define PMAPDEBUG_KENTER 0x2000
427: #define PMAPDEBUG_KREMOVE 0x4000
428: #define PMAPDEBUG_REMOVE 0x8000
1.38 sanjayl 429:
1.1 matt 430: unsigned int pmapdebug = 0;
1.38 sanjayl 431:
1.85 matt 432: # define DPRINTF(x, ...) printf(x, __VA_ARGS__)
433: # define DPRINTFN(n, x, ...) do if (pmapdebug & PMAPDEBUG_ ## n) printf(x, __VA_ARGS__); while (0)
1.1 matt 434: #else
1.85 matt 435: # define DPRINTF(x, ...) do { } while (0)
436: # define DPRINTFN(n, x, ...) do { } while (0)
1.1 matt 437: #endif
438:
439:
440: #ifdef PMAPCOUNTERS
441: /*
442: * From pmap_subr.c
443: */
1.53 garbled 444: extern struct evcnt pmap_evcnt_mappings;
445: extern struct evcnt pmap_evcnt_unmappings;
446:
447: extern struct evcnt pmap_evcnt_kernel_mappings;
448: extern struct evcnt pmap_evcnt_kernel_unmappings;
449:
450: extern struct evcnt pmap_evcnt_mappings_replaced;
451:
452: extern struct evcnt pmap_evcnt_exec_mappings;
453: extern struct evcnt pmap_evcnt_exec_cached;
454:
455: extern struct evcnt pmap_evcnt_exec_synced;
456: extern struct evcnt pmap_evcnt_exec_synced_clear_modify;
457: extern struct evcnt pmap_evcnt_exec_synced_pvo_remove;
458:
459: extern struct evcnt pmap_evcnt_exec_uncached_page_protect;
460: extern struct evcnt pmap_evcnt_exec_uncached_clear_modify;
461: extern struct evcnt pmap_evcnt_exec_uncached_zero_page;
462: extern struct evcnt pmap_evcnt_exec_uncached_copy_page;
463: extern struct evcnt pmap_evcnt_exec_uncached_pvo_remove;
464:
465: extern struct evcnt pmap_evcnt_updates;
466: extern struct evcnt pmap_evcnt_collects;
467: extern struct evcnt pmap_evcnt_copies;
468:
469: extern struct evcnt pmap_evcnt_ptes_spilled;
470: extern struct evcnt pmap_evcnt_ptes_unspilled;
471: extern struct evcnt pmap_evcnt_ptes_evicted;
472:
473: extern struct evcnt pmap_evcnt_ptes_primary[8];
474: extern struct evcnt pmap_evcnt_ptes_secondary[8];
475: extern struct evcnt pmap_evcnt_ptes_removed;
476: extern struct evcnt pmap_evcnt_ptes_changed;
477: extern struct evcnt pmap_evcnt_pvos_reclaimed;
478: extern struct evcnt pmap_evcnt_pvos_failed;
479:
1.1 matt 480: extern struct evcnt pmap_evcnt_zeroed_pages;
481: extern struct evcnt pmap_evcnt_copied_pages;
482: extern struct evcnt pmap_evcnt_idlezeroed_pages;
1.26 matt 483:
1.53 garbled 484: #define PMAPCOUNT(ev) ((pmap_evcnt_ ## ev).ev_count++)
485: #define PMAPCOUNT2(ev) ((ev).ev_count++)
1.1 matt 486: #else
487: #define PMAPCOUNT(ev) ((void) 0)
488: #define PMAPCOUNT2(ev) ((void) 0)
489: #endif
490:
1.35 perry 491: #define TLBIE(va) __asm volatile("tlbie %0" :: "r"(va))
1.38 sanjayl 492:
493: /* XXXSL: this needs to be moved to assembler */
494: #define TLBIEL(va) __asm __volatile("tlbie %0" :: "r"(va))
495:
1.87 kiyohara 496: #ifdef MD_TLBSYNC
497: #define TLBSYNC() MD_TLBSYNC()
498: #else
1.35 perry 499: #define TLBSYNC() __asm volatile("tlbsync")
1.87 kiyohara 500: #endif
1.35 perry 501: #define SYNC() __asm volatile("sync")
502: #define EIEIO() __asm volatile("eieio")
1.57 matt 503: #define DCBST(va) __asm __volatile("dcbst 0,%0" :: "r"(va))
1.1 matt 504: #define MFMSR() mfmsr()
505: #define MTMSR(psl) mtmsr(psl)
506: #define MFPVR() mfpvr()
507: #define MFSRIN(va) mfsrin(va)
508: #define MFTB() mfrtcltbl()
509:
1.92 joerg 510: #if defined(DDB) && !defined(PMAP_OEA64)
1.35 perry 511: static inline register_t
1.1 matt 512: mfsrin(vaddr_t va)
513: {
1.2 matt 514: register_t sr;
1.35 perry 515: __asm volatile ("mfsrin %0,%1" : "=r"(sr) : "r"(va));
1.1 matt 516: return sr;
517: }
1.92 joerg 518: #endif /* DDB && !PMAP_OEA64 */
1.38 sanjayl 519:
1.53 garbled 520: #if defined (PMAP_OEA64_BRIDGE)
1.38 sanjayl 521: extern void mfmsr64 (register64_t *result);
1.53 garbled 522: #endif /* PMAP_OEA64_BRIDGE */
1.38 sanjayl 523:
1.50 ad 524: #define PMAP_LOCK() KERNEL_LOCK(1, NULL)
525: #define PMAP_UNLOCK() KERNEL_UNLOCK_ONE(NULL)
1.1 matt 526:
1.35 perry 527: static inline register_t
1.1 matt 528: pmap_interrupts_off(void)
529: {
1.2 matt 530: register_t msr = MFMSR();
1.1 matt 531: if (msr & PSL_EE)
532: MTMSR(msr & ~PSL_EE);
533: return msr;
534: }
535:
536: static void
1.2 matt 537: pmap_interrupts_restore(register_t msr)
1.1 matt 538: {
539: if (msr & PSL_EE)
540: MTMSR(msr);
541: }
542:
1.35 perry 543: static inline u_int32_t
1.1 matt 544: mfrtcltbl(void)
545: {
1.55 garbled 546: #ifdef PPC_OEA601
1.1 matt 547: if ((MFPVR() >> 16) == MPC601)
548: return (mfrtcl() >> 7);
549: else
1.55 garbled 550: #endif
1.1 matt 551: return (mftbl());
552: }
553:
554: /*
555: * These small routines may have to be replaced,
556: * if/when we support processors other that the 604.
557: */
558:
559: void
560: tlbia(void)
561: {
1.47 macallan 562: char *i;
1.1 matt 563:
564: SYNC();
1.53 garbled 565: #if defined(PMAP_OEA)
1.1 matt 566: /*
567: * Why not use "tlbia"? Because not all processors implement it.
568: *
1.20 wiz 569: * This needs to be a per-CPU callback to do the appropriate thing
1.1 matt 570: * for the CPU. XXX
571: */
1.47 macallan 572: for (i = 0; i < (char *)0x00040000; i += 0x00001000) {
1.1 matt 573: TLBIE(i);
574: EIEIO();
575: SYNC();
576: }
1.53 garbled 577: #elif defined (PMAP_OEA64) || defined (PMAP_OEA64_BRIDGE)
1.38 sanjayl 578: /* This is specifically for the 970, 970UM v1.6 pp. 140. */
1.51 garbled 579: for (i = 0; i <= (char *)0xFF000; i += 0x00001000) {
1.38 sanjayl 580: TLBIEL(i);
581: EIEIO();
582: SYNC();
583: }
584: #endif
1.1 matt 585: TLBSYNC();
586: SYNC();
587: }
588:
1.35 perry 589: static inline register_t
1.2 matt 590: va_to_vsid(const struct pmap *pm, vaddr_t addr)
1.1 matt 591: {
1.53 garbled 592: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.38 sanjayl 593: return (pm->pm_sr[addr >> ADDR_SR_SHFT] & SR_VSID) >> SR_VSID_SHFT;
1.53 garbled 594: #else /* PMAP_OEA64 */
1.18 matt 595: #if 0
596: const struct ste *ste;
597: register_t hash;
598: int i;
599:
600: hash = (addr >> ADDR_ESID_SHFT) & ADDR_ESID_HASH;
601:
602: /*
603: * Try the primary group first
604: */
605: ste = pm->pm_stes[hash].stes;
606: for (i = 0; i < 8; i++, ste++) {
607: if (ste->ste_hi & STE_V) &&
608: (addr & ~(ADDR_POFF|ADDR_PIDX)) == (ste->ste_hi & STE_ESID))
609: return ste;
610: }
611:
612: /*
613: * Then the secondary group.
614: */
615: ste = pm->pm_stes[hash ^ ADDR_ESID_HASH].stes;
616: for (i = 0; i < 8; i++, ste++) {
617: if (ste->ste_hi & STE_V) &&
618: (addr & ~(ADDR_POFF|ADDR_PIDX)) == (ste->ste_hi & STE_ESID))
619: return addr;
620: }
621:
622: return NULL;
623: #else
624: /*
625: * Rather than searching the STE groups for the VSID, we know
626: * how we generate that from the ESID and so do that.
627: */
628: return VSID_MAKE(addr >> ADDR_SR_SHFT, pm->pm_vsid) >> SR_VSID_SHFT;
629: #endif
1.53 garbled 630: #endif /* PMAP_OEA */
1.1 matt 631: }
632:
1.35 perry 633: static inline register_t
1.2 matt 634: va_to_pteg(const struct pmap *pm, vaddr_t addr)
1.1 matt 635: {
1.2 matt 636: register_t hash;
637:
638: hash = va_to_vsid(pm, addr) ^ ((addr & ADDR_PIDX) >> ADDR_PIDX_SHFT);
1.1 matt 639: return hash & pmap_pteg_mask;
640: }
641:
642: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
643: /*
644: * Given a PTE in the page table, calculate the VADDR that hashes to it.
645: * The only bit of magic is that the top 4 bits of the address doesn't
646: * technically exist in the PTE. But we know we reserved 4 bits of the
647: * VSID for it so that's how we get it.
648: */
649: static vaddr_t
1.2 matt 650: pmap_pte_to_va(volatile const struct pte *pt)
1.1 matt 651: {
652: vaddr_t va;
653: uintptr_t ptaddr = (uintptr_t) pt;
654:
655: if (pt->pte_hi & PTE_HID)
1.2 matt 656: ptaddr ^= (pmap_pteg_mask * sizeof(struct pteg));
1.1 matt 657:
1.18 matt 658: /* PPC Bits 10-19 PPC64 Bits 42-51 */
1.53 garbled 659: #if defined(PMAP_OEA)
1.4 matt 660: va = ((pt->pte_hi >> PTE_VSID_SHFT) ^ (ptaddr / sizeof(struct pteg))) & 0x3ff;
1.53 garbled 661: #elif defined (PMAP_OEA64) || defined (PMAP_OEA64_BRIDGE)
1.38 sanjayl 662: va = ((pt->pte_hi >> PTE_VSID_SHFT) ^ (ptaddr / sizeof(struct pteg))) & 0x7ff;
663: #endif
1.1 matt 664: va <<= ADDR_PIDX_SHFT;
665:
1.18 matt 666: /* PPC Bits 4-9 PPC64 Bits 36-41 */
1.1 matt 667: va |= (pt->pte_hi & PTE_API) << ADDR_API_SHFT;
668:
1.53 garbled 669: #if defined(PMAP_OEA64)
1.18 matt 670: /* PPC63 Bits 0-35 */
671: /* va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT; */
1.53 garbled 672: #elif defined(PMAP_OEA) || defined(PMAP_OEA64_BRIDGE)
1.1 matt 673: /* PPC Bits 0-3 */
674: va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT;
1.18 matt 675: #endif
1.1 matt 676:
677: return va;
678: }
679: #endif
680:
1.35 perry 681: static inline struct pvo_head *
1.1 matt 682: pa_to_pvoh(paddr_t pa, struct vm_page **pg_p)
683: {
684: struct vm_page *pg;
1.72 uebayasi 685: struct vm_page_md *md;
1.1 matt 686:
687: pg = PHYS_TO_VM_PAGE(pa);
688: if (pg_p != NULL)
689: *pg_p = pg;
690: if (pg == NULL)
691: return &pmap_pvo_unmanaged;
1.72 uebayasi 692: md = VM_PAGE_TO_MD(pg);
693: return &md->mdpg_pvoh;
1.1 matt 694: }
695:
1.35 perry 696: static inline struct pvo_head *
1.1 matt 697: vm_page_to_pvoh(struct vm_page *pg)
698: {
1.72 uebayasi 699: struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
700:
701: return &md->mdpg_pvoh;
1.1 matt 702: }
703:
704:
1.35 perry 705: static inline void
1.1 matt 706: pmap_attr_clear(struct vm_page *pg, int ptebit)
707: {
1.72 uebayasi 708: struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
709:
710: md->mdpg_attrs &= ~ptebit;
1.1 matt 711: }
712:
1.35 perry 713: static inline int
1.1 matt 714: pmap_attr_fetch(struct vm_page *pg)
715: {
1.72 uebayasi 716: struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
717:
718: return md->mdpg_attrs;
1.1 matt 719: }
720:
1.35 perry 721: static inline void
1.1 matt 722: pmap_attr_save(struct vm_page *pg, int ptebit)
723: {
1.72 uebayasi 724: struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
725:
726: md->mdpg_attrs |= ptebit;
1.1 matt 727: }
728:
1.35 perry 729: static inline int
1.2 matt 730: pmap_pte_compare(const volatile struct pte *pt, const struct pte *pvo_pt)
1.1 matt 731: {
732: if (pt->pte_hi == pvo_pt->pte_hi
733: #if 0
734: && ((pt->pte_lo ^ pvo_pt->pte_lo) &
735: ~(PTE_REF|PTE_CHG)) == 0
736: #endif
737: )
738: return 1;
739: return 0;
740: }
741:
1.35 perry 742: static inline void
1.2 matt 743: pmap_pte_create(struct pte *pt, const struct pmap *pm, vaddr_t va, register_t pte_lo)
1.1 matt 744: {
745: /*
746: * Construct the PTE. Default to IMB initially. Valid bit
747: * only gets set when the real pte is set in memory.
748: *
749: * Note: Don't set the valid bit for correct operation of tlb update.
750: */
1.53 garbled 751: #if defined(PMAP_OEA)
1.2 matt 752: pt->pte_hi = (va_to_vsid(pm, va) << PTE_VSID_SHFT)
753: | (((va & ADDR_PIDX) >> (ADDR_API_SHFT - PTE_API_SHFT)) & PTE_API);
1.1 matt 754: pt->pte_lo = pte_lo;
1.79 matt 755: #elif defined (PMAP_OEA64_BRIDGE) || defined (PMAP_OEA64)
1.38 sanjayl 756: pt->pte_hi = ((u_int64_t)va_to_vsid(pm, va) << PTE_VSID_SHFT)
757: | (((va & ADDR_PIDX) >> (ADDR_API_SHFT - PTE_API_SHFT)) & PTE_API);
758: pt->pte_lo = (u_int64_t) pte_lo;
1.53 garbled 759: #endif /* PMAP_OEA */
1.1 matt 760: }
761:
1.35 perry 762: static inline void
1.2 matt 763: pmap_pte_synch(volatile struct pte *pt, struct pte *pvo_pt)
1.1 matt 764: {
765: pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF|PTE_CHG);
766: }
767:
1.35 perry 768: static inline void
1.2 matt 769: pmap_pte_clear(volatile struct pte *pt, vaddr_t va, int ptebit)
1.1 matt 770: {
771: /*
772: * As shown in Section 7.6.3.2.3
773: */
774: pt->pte_lo &= ~ptebit;
775: TLBIE(va);
776: SYNC();
777: EIEIO();
778: TLBSYNC();
779: SYNC();
1.57 matt 780: #ifdef MULTIPROCESSOR
781: DCBST(pt);
782: #endif
1.1 matt 783: }
784:
1.35 perry 785: static inline void
1.2 matt 786: pmap_pte_set(volatile struct pte *pt, struct pte *pvo_pt)
1.1 matt 787: {
788: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
789: if (pvo_pt->pte_hi & PTE_VALID)
790: panic("pte_set: setting an already valid pte %p", pvo_pt);
791: #endif
792: pvo_pt->pte_hi |= PTE_VALID;
1.38 sanjayl 793:
1.1 matt 794: /*
795: * Update the PTE as defined in section 7.6.3.1
796: * Note that the REF/CHG bits are from pvo_pt and thus should
797: * have been saved so this routine can restore them (if desired).
798: */
799: pt->pte_lo = pvo_pt->pte_lo;
800: EIEIO();
801: pt->pte_hi = pvo_pt->pte_hi;
1.38 sanjayl 802: TLBSYNC();
1.1 matt 803: SYNC();
1.57 matt 804: #ifdef MULTIPROCESSOR
805: DCBST(pt);
806: #endif
1.1 matt 807: pmap_pte_valid++;
808: }
809:
1.35 perry 810: static inline void
1.2 matt 811: pmap_pte_unset(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va)
1.1 matt 812: {
813: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
814: if ((pvo_pt->pte_hi & PTE_VALID) == 0)
815: panic("pte_unset: attempt to unset an inactive pte#1 %p/%p", pvo_pt, pt);
816: if ((pt->pte_hi & PTE_VALID) == 0)
817: panic("pte_unset: attempt to unset an inactive pte#2 %p/%p", pvo_pt, pt);
818: #endif
819:
820: pvo_pt->pte_hi &= ~PTE_VALID;
821: /*
822: * Force the ref & chg bits back into the PTEs.
823: */
824: SYNC();
825: /*
826: * Invalidate the pte ... (Section 7.6.3.3)
827: */
828: pt->pte_hi &= ~PTE_VALID;
829: SYNC();
830: TLBIE(va);
831: SYNC();
832: EIEIO();
833: TLBSYNC();
834: SYNC();
835: /*
836: * Save the ref & chg bits ...
837: */
838: pmap_pte_synch(pt, pvo_pt);
839: pmap_pte_valid--;
840: }
841:
1.35 perry 842: static inline void
1.2 matt 843: pmap_pte_change(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va)
1.1 matt 844: {
845: /*
846: * Invalidate the PTE
847: */
848: pmap_pte_unset(pt, pvo_pt, va);
849: pmap_pte_set(pt, pvo_pt);
850: }
851:
852: /*
853: * Try to insert the PTE @ *pvo_pt into the pmap_pteg_table at ptegidx
854: * (either primary or secondary location).
855: *
856: * Note: both the destination and source PTEs must not have PTE_VALID set.
857: */
858:
1.53 garbled 859: static int
1.2 matt 860: pmap_pte_insert(int ptegidx, struct pte *pvo_pt)
1.1 matt 861: {
1.2 matt 862: volatile struct pte *pt;
1.1 matt 863: int i;
864:
865: #if defined(DEBUG)
1.85 matt 866: DPRINTFN(PTE, "pmap_pte_insert: idx %#x, pte %#" _PRIxpte " %#" _PRIxpte "\n",
867: ptegidx, pvo_pt->pte_hi, pvo_pt->pte_lo);
1.1 matt 868: #endif
869: /*
870: * First try primary hash.
871: */
872: for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
873: if ((pt->pte_hi & PTE_VALID) == 0) {
874: pvo_pt->pte_hi &= ~PTE_HID;
875: pmap_pte_set(pt, pvo_pt);
876: return i;
877: }
878: }
879:
880: /*
881: * Now try secondary hash.
882: */
883: ptegidx ^= pmap_pteg_mask;
884: for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
885: if ((pt->pte_hi & PTE_VALID) == 0) {
886: pvo_pt->pte_hi |= PTE_HID;
887: pmap_pte_set(pt, pvo_pt);
888: return i;
889: }
890: }
891: return -1;
892: }
893:
894: /*
895: * Spill handler.
896: *
897: * Tries to spill a page table entry from the overflow area.
898: * This runs in either real mode (if dealing with a exception spill)
899: * or virtual mode when dealing with manually spilling one of the
900: * kernel's pte entries. In either case, interrupts are already
901: * disabled.
902: */
1.14 chs 903:
1.1 matt 904: int
1.44 thorpej 905: pmap_pte_spill(struct pmap *pm, vaddr_t addr, bool exec)
1.1 matt 906: {
907: struct pvo_entry *source_pvo, *victim_pvo, *next_pvo;
908: struct pvo_entry *pvo;
1.15 dyoung 909: /* XXX: gcc -- vpvoh is always set at either *1* or *2* */
910: struct pvo_tqhead *pvoh, *vpvoh = NULL;
1.1 matt 911: int ptegidx, i, j;
1.2 matt 912: volatile struct pteg *pteg;
913: volatile struct pte *pt;
1.1 matt 914:
1.50 ad 915: PMAP_LOCK();
916:
1.2 matt 917: ptegidx = va_to_pteg(pm, addr);
1.1 matt 918:
919: /*
920: * Have to substitute some entry. Use the primary hash for this.
1.12 matt 921: * Use low bits of timebase as random generator. Make sure we are
922: * not picking a kernel pte for replacement.
1.1 matt 923: */
924: pteg = &pmap_pteg_table[ptegidx];
925: i = MFTB() & 7;
1.12 matt 926: for (j = 0; j < 8; j++) {
927: pt = &pteg->pt[i];
1.53 garbled 928: if ((pt->pte_hi & PTE_VALID) == 0)
929: break;
930: if (VSID_TO_HASH((pt->pte_hi & PTE_VSID) >> PTE_VSID_SHFT)
931: < PHYSMAP_VSIDBITS)
1.12 matt 932: break;
933: i = (i + 1) & 7;
934: }
935: KASSERT(j < 8);
1.1 matt 936:
937: source_pvo = NULL;
938: victim_pvo = NULL;
939: pvoh = &pmap_pvo_table[ptegidx];
940: TAILQ_FOREACH(pvo, pvoh, pvo_olink) {
941:
942: /*
943: * We need to find pvo entry for this address...
944: */
945: PMAP_PVO_CHECK(pvo); /* sanity check */
946:
947: /*
948: * If we haven't found the source and we come to a PVO with
949: * a valid PTE, then we know we can't find it because all
950: * evicted PVOs always are first in the list.
951: */
952: if (source_pvo == NULL && (pvo->pvo_pte.pte_hi & PTE_VALID))
953: break;
1.2 matt 954: if (source_pvo == NULL && pm == pvo->pvo_pmap &&
955: addr == PVO_VADDR(pvo)) {
1.1 matt 956:
957: /*
958: * Now we have found the entry to be spilled into the
959: * pteg. Attempt to insert it into the page table.
960: */
961: j = pmap_pte_insert(ptegidx, &pvo->pvo_pte);
962: if (j >= 0) {
963: PVO_PTEGIDX_SET(pvo, j);
964: PMAP_PVO_CHECK(pvo); /* sanity check */
1.12 matt 965: PVO_WHERE(pvo, SPILL_INSERT);
1.1 matt 966: pvo->pvo_pmap->pm_evictions--;
967: PMAPCOUNT(ptes_spilled);
968: PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID)
969: ? pmap_evcnt_ptes_secondary
970: : pmap_evcnt_ptes_primary)[j]);
971:
972: /*
973: * Since we keep the evicted entries at the
974: * from of the PVO list, we need move this
975: * (now resident) PVO after the evicted
976: * entries.
977: */
978: next_pvo = TAILQ_NEXT(pvo, pvo_olink);
979:
980: /*
1.5 matt 981: * If we don't have to move (either we were the
982: * last entry or the next entry was valid),
1.1 matt 983: * don't change our position. Otherwise
984: * move ourselves to the tail of the queue.
985: */
986: if (next_pvo != NULL &&
987: !(next_pvo->pvo_pte.pte_hi & PTE_VALID)) {
988: TAILQ_REMOVE(pvoh, pvo, pvo_olink);
989: TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink);
990: }
1.50 ad 991: PMAP_UNLOCK();
1.1 matt 992: return 1;
993: }
994: source_pvo = pvo;
1.39 matt 995: if (exec && !PVO_EXECUTABLE_P(source_pvo)) {
1.96 rin 996: PMAP_UNLOCK();
1.14 chs 997: return 0;
998: }
1.1 matt 999: if (victim_pvo != NULL)
1000: break;
1001: }
1002:
1003: /*
1004: * We also need the pvo entry of the victim we are replacing
1005: * so save the R & C bits of the PTE.
1006: */
1007: if ((pt->pte_hi & PTE_HID) == 0 && victim_pvo == NULL &&
1008: pmap_pte_compare(pt, &pvo->pvo_pte)) {
1.15 dyoung 1009: vpvoh = pvoh; /* *1* */
1.1 matt 1010: victim_pvo = pvo;
1011: if (source_pvo != NULL)
1012: break;
1013: }
1014: }
1015:
1016: if (source_pvo == NULL) {
1017: PMAPCOUNT(ptes_unspilled);
1.50 ad 1018: PMAP_UNLOCK();
1.1 matt 1019: return 0;
1020: }
1021:
1022: if (victim_pvo == NULL) {
1023: if ((pt->pte_hi & PTE_HID) == 0)
1024: panic("pmap_pte_spill: victim p-pte (%p) has "
1025: "no pvo entry!", pt);
1026:
1027: /*
1028: * If this is a secondary PTE, we need to search
1029: * its primary pvo bucket for the matching PVO.
1030: */
1.15 dyoung 1031: vpvoh = &pmap_pvo_table[ptegidx ^ pmap_pteg_mask]; /* *2* */
1.1 matt 1032: TAILQ_FOREACH(pvo, vpvoh, pvo_olink) {
1033: PMAP_PVO_CHECK(pvo); /* sanity check */
1034:
1035: /*
1036: * We also need the pvo entry of the victim we are
1037: * replacing so save the R & C bits of the PTE.
1038: */
1039: if (pmap_pte_compare(pt, &pvo->pvo_pte)) {
1040: victim_pvo = pvo;
1041: break;
1042: }
1043: }
1044: if (victim_pvo == NULL)
1045: panic("pmap_pte_spill: victim s-pte (%p) has "
1046: "no pvo entry!", pt);
1047: }
1048:
1049: /*
1.12 matt 1050: * The victim should be not be a kernel PVO/PTE entry.
1051: */
1052: KASSERT(victim_pvo->pvo_pmap != pmap_kernel());
1053: KASSERT(PVO_PTEGIDX_ISSET(victim_pvo));
1054: KASSERT(PVO_PTEGIDX_GET(victim_pvo) == i);
1055:
1056: /*
1.1 matt 1057: * We are invalidating the TLB entry for the EA for the
1058: * we are replacing even though its valid; If we don't
1059: * we lose any ref/chg bit changes contained in the TLB
1060: * entry.
1061: */
1062: source_pvo->pvo_pte.pte_hi &= ~PTE_HID;
1063:
1064: /*
1065: * To enforce the PVO list ordering constraint that all
1066: * evicted entries should come before all valid entries,
1067: * move the source PVO to the tail of its list and the
1068: * victim PVO to the head of its list (which might not be
1069: * the same list, if the victim was using the secondary hash).
1070: */
1071: TAILQ_REMOVE(pvoh, source_pvo, pvo_olink);
1072: TAILQ_INSERT_TAIL(pvoh, source_pvo, pvo_olink);
1073: TAILQ_REMOVE(vpvoh, victim_pvo, pvo_olink);
1074: TAILQ_INSERT_HEAD(vpvoh, victim_pvo, pvo_olink);
1075: pmap_pte_unset(pt, &victim_pvo->pvo_pte, victim_pvo->pvo_vaddr);
1076: pmap_pte_set(pt, &source_pvo->pvo_pte);
1077: victim_pvo->pvo_pmap->pm_evictions++;
1078: source_pvo->pvo_pmap->pm_evictions--;
1.12 matt 1079: PVO_WHERE(victim_pvo, SPILL_UNSET);
1080: PVO_WHERE(source_pvo, SPILL_SET);
1.1 matt 1081:
1082: PVO_PTEGIDX_CLR(victim_pvo);
1083: PVO_PTEGIDX_SET(source_pvo, i);
1084: PMAPCOUNT2(pmap_evcnt_ptes_primary[i]);
1085: PMAPCOUNT(ptes_spilled);
1086: PMAPCOUNT(ptes_evicted);
1087: PMAPCOUNT(ptes_removed);
1088:
1089: PMAP_PVO_CHECK(victim_pvo);
1090: PMAP_PVO_CHECK(source_pvo);
1.50 ad 1091:
1092: PMAP_UNLOCK();
1.1 matt 1093: return 1;
1094: }
1095:
1096: /*
1097: * Restrict given range to physical memory
1098: */
1099: void
1100: pmap_real_memory(paddr_t *start, psize_t *size)
1101: {
1102: struct mem_region *mp;
1103:
1104: for (mp = mem; mp->size; mp++) {
1105: if (*start + *size > mp->start
1106: && *start < mp->start + mp->size) {
1107: if (*start < mp->start) {
1108: *size -= mp->start - *start;
1109: *start = mp->start;
1110: }
1111: if (*start + *size > mp->start + mp->size)
1112: *size = mp->start + mp->size - *start;
1113: return;
1114: }
1115: }
1116: *size = 0;
1117: }
1118:
1119: /*
1120: * Initialize anything else for pmap handling.
1121: * Called during vm_init().
1122: */
1123: void
1124: pmap_init(void)
1125: {
1126: pool_init(&pmap_mpvo_pool, sizeof(struct pvo_entry),
1127: sizeof(struct pvo_entry), 0, 0, "pmap_mpvopl",
1.48 ad 1128: &pmap_pool_mallocator, IPL_NONE);
1.1 matt 1129:
1130: pool_setlowat(&pmap_mpvo_pool, 1008);
1131:
1132: pmap_initialized = 1;
1133:
1134: }
1135:
1136: /*
1.10 thorpej 1137: * How much virtual space does the kernel get?
1138: */
1139: void
1140: pmap_virtual_space(vaddr_t *start, vaddr_t *end)
1141: {
1142: /*
1143: * For now, reserve one segment (minus some overhead) for kernel
1144: * virtual memory
1145: */
1146: *start = VM_MIN_KERNEL_ADDRESS;
1147: *end = VM_MAX_KERNEL_ADDRESS;
1148: }
1149:
1150: /*
1.1 matt 1151: * Allocate, initialize, and return a new physical map.
1152: */
1153: pmap_t
1154: pmap_create(void)
1155: {
1156: pmap_t pm;
1.38 sanjayl 1157:
1.1 matt 1158: pm = pool_get(&pmap_pool, PR_WAITOK);
1.84 matt 1159: KASSERT((vaddr_t)pm < VM_MIN_KERNEL_ADDRESS);
1.46 christos 1160: memset((void *)pm, 0, sizeof *pm);
1.1 matt 1161: pmap_pinit(pm);
1162:
1.85 matt 1163: DPRINTFN(CREATE, "pmap_create: pm %p:\n"
1.54 mlelstv 1164: "\t%#" _PRIsr " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr
1165: " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr "\n"
1166: "\t%#" _PRIsr " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr
1167: " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr " %#" _PRIsr "\n",
1168: pm,
1169: pm->pm_sr[0], pm->pm_sr[1],
1170: pm->pm_sr[2], pm->pm_sr[3],
1171: pm->pm_sr[4], pm->pm_sr[5],
1172: pm->pm_sr[6], pm->pm_sr[7],
1173: pm->pm_sr[8], pm->pm_sr[9],
1174: pm->pm_sr[10], pm->pm_sr[11],
1175: pm->pm_sr[12], pm->pm_sr[13],
1.85 matt 1176: pm->pm_sr[14], pm->pm_sr[15]);
1.1 matt 1177: return pm;
1178: }
1179:
1180: /*
1181: * Initialize a preallocated and zeroed pmap structure.
1182: */
1183: void
1184: pmap_pinit(pmap_t pm)
1185: {
1.2 matt 1186: register_t entropy = MFTB();
1187: register_t mask;
1188: int i;
1.1 matt 1189:
1190: /*
1191: * Allocate some segment registers for this pmap.
1192: */
1193: pm->pm_refs = 1;
1.50 ad 1194: PMAP_LOCK();
1.2 matt 1195: for (i = 0; i < NPMAPS; i += VSID_NBPW) {
1196: static register_t pmap_vsidcontext;
1197: register_t hash;
1198: unsigned int n;
1.1 matt 1199:
1200: /* Create a new value by multiplying by a prime adding in
1201: * entropy from the timebase register. This is to make the
1202: * VSID more random so that the PT Hash function collides
1203: * less often. (note that the prime causes gcc to do shifts
1204: * instead of a multiply)
1205: */
1206: pmap_vsidcontext = (pmap_vsidcontext * 0x1105) + entropy;
1207: hash = pmap_vsidcontext & (NPMAPS - 1);
1.23 aymeric 1208: if (hash == 0) { /* 0 is special, avoid it */
1209: entropy += 0xbadf00d;
1.1 matt 1210: continue;
1.23 aymeric 1211: }
1.1 matt 1212: n = hash >> 5;
1.2 matt 1213: mask = 1L << (hash & (VSID_NBPW-1));
1214: hash = pmap_vsidcontext;
1.1 matt 1215: if (pmap_vsid_bitmap[n] & mask) { /* collision? */
1216: /* anything free in this bucket? */
1.2 matt 1217: if (~pmap_vsid_bitmap[n] == 0) {
1.23 aymeric 1218: entropy = hash ^ (hash >> 16);
1.1 matt 1219: continue;
1220: }
1221: i = ffs(~pmap_vsid_bitmap[n]) - 1;
1.2 matt 1222: mask = 1L << i;
1223: hash &= ~(VSID_NBPW-1);
1.1 matt 1224: hash |= i;
1225: }
1.18 matt 1226: hash &= PTE_VSID >> PTE_VSID_SHFT;
1.1 matt 1227: pmap_vsid_bitmap[n] |= mask;
1.18 matt 1228: pm->pm_vsid = hash;
1.53 garbled 1229: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1 matt 1230: for (i = 0; i < 16; i++)
1.14 chs 1231: pm->pm_sr[i] = VSID_MAKE(i, hash) | SR_PRKEY |
1232: SR_NOEXEC;
1.18 matt 1233: #endif
1.50 ad 1234: PMAP_UNLOCK();
1.1 matt 1235: return;
1236: }
1.50 ad 1237: PMAP_UNLOCK();
1.1 matt 1238: panic("pmap_pinit: out of segments");
1239: }
1240:
1241: /*
1242: * Add a reference to the given pmap.
1243: */
1244: void
1245: pmap_reference(pmap_t pm)
1246: {
1.50 ad 1247: atomic_inc_uint(&pm->pm_refs);
1.1 matt 1248: }
1249:
1250: /*
1251: * Retire the given pmap from service.
1252: * Should only be called if the map contains no valid mappings.
1253: */
1254: void
1255: pmap_destroy(pmap_t pm)
1256: {
1.50 ad 1257: if (atomic_dec_uint_nv(&pm->pm_refs) == 0) {
1.1 matt 1258: pmap_release(pm);
1259: pool_put(&pmap_pool, pm);
1260: }
1261: }
1262:
1263: /*
1264: * Release any resources held by the given physical map.
1265: * Called when a pmap initialized by pmap_pinit is being released.
1266: */
1267: void
1268: pmap_release(pmap_t pm)
1269: {
1270: int idx, mask;
1.39 matt 1271:
1272: KASSERT(pm->pm_stats.resident_count == 0);
1273: KASSERT(pm->pm_stats.wired_count == 0);
1.1 matt 1274:
1.50 ad 1275: PMAP_LOCK();
1.1 matt 1276: if (pm->pm_sr[0] == 0)
1277: panic("pmap_release");
1.22 aymeric 1278: idx = pm->pm_vsid & (NPMAPS-1);
1.1 matt 1279: mask = 1 << (idx % VSID_NBPW);
1280: idx /= VSID_NBPW;
1.22 aymeric 1281:
1282: KASSERT(pmap_vsid_bitmap[idx] & mask);
1.1 matt 1283: pmap_vsid_bitmap[idx] &= ~mask;
1.50 ad 1284: PMAP_UNLOCK();
1.1 matt 1285: }
1286:
1287: /*
1288: * Copy the range specified by src_addr/len
1289: * from the source map to the range dst_addr/len
1290: * in the destination map.
1291: *
1292: * This routine is only advisory and need not do anything.
1293: */
1294: void
1295: pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr,
1296: vsize_t len, vaddr_t src_addr)
1297: {
1298: PMAPCOUNT(copies);
1299: }
1300:
1301: /*
1302: * Require that all active physical maps contain no
1303: * incorrect entries NOW.
1304: */
1305: void
1306: pmap_update(struct pmap *pmap)
1307: {
1308: PMAPCOUNT(updates);
1309: TLBSYNC();
1310: }
1311:
1.35 perry 1312: static inline int
1.1 matt 1313: pmap_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx)
1314: {
1315: int pteidx;
1316: /*
1317: * We can find the actual pte entry without searching by
1318: * grabbing the PTEG index from 3 unused bits in pte_lo[11:9]
1319: * and by noticing the HID bit.
1320: */
1321: pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo);
1322: if (pvo->pvo_pte.pte_hi & PTE_HID)
1323: pteidx ^= pmap_pteg_mask * 8;
1324: return pteidx;
1325: }
1326:
1.2 matt 1327: volatile struct pte *
1.1 matt 1328: pmap_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
1329: {
1.2 matt 1330: volatile struct pte *pt;
1.1 matt 1331:
1332: #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK)
1333: if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0)
1334: return NULL;
1335: #endif
1336:
1337: /*
1338: * If we haven't been supplied the ptegidx, calculate it.
1339: */
1340: if (pteidx == -1) {
1341: int ptegidx;
1.2 matt 1342: ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr);
1.1 matt 1343: pteidx = pmap_pvo_pte_index(pvo, ptegidx);
1344: }
1345:
1346: pt = &pmap_pteg_table[pteidx >> 3].pt[pteidx & 7];
1347:
1348: #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK)
1349: return pt;
1350: #else
1351: if ((pvo->pvo_pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) {
1352: panic("pmap_pvo_to_pte: pvo %p: has valid pte in "
1353: "pvo but no valid pte index", pvo);
1354: }
1355: if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) {
1356: panic("pmap_pvo_to_pte: pvo %p: has valid pte index in "
1357: "pvo but no valid pte", pvo);
1358: }
1359:
1360: if ((pt->pte_hi ^ (pvo->pvo_pte.pte_hi & ~PTE_VALID)) == PTE_VALID) {
1361: if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) {
1362: #if defined(DEBUG) || defined(PMAPCHECK)
1363: pmap_pte_print(pt);
1364: #endif
1365: panic("pmap_pvo_to_pte: pvo %p: has valid pte in "
1366: "pmap_pteg_table %p but invalid in pvo",
1367: pvo, pt);
1368: }
1369: if (((pt->pte_lo ^ pvo->pvo_pte.pte_lo) & ~(PTE_CHG|PTE_REF)) != 0) {
1370: #if defined(DEBUG) || defined(PMAPCHECK)
1371: pmap_pte_print(pt);
1372: #endif
1373: panic("pmap_pvo_to_pte: pvo %p: pvo pte does "
1374: "not match pte %p in pmap_pteg_table",
1375: pvo, pt);
1376: }
1377: return pt;
1378: }
1379:
1380: if (pvo->pvo_pte.pte_hi & PTE_VALID) {
1381: #if defined(DEBUG) || defined(PMAPCHECK)
1382: pmap_pte_print(pt);
1383: #endif
1.12 matt 1384: panic("pmap_pvo_to_pte: pvo %p: has nomatching pte %p in "
1.1 matt 1385: "pmap_pteg_table but valid in pvo", pvo, pt);
1386: }
1387: return NULL;
1388: #endif /* !(!DIAGNOSTIC && !DEBUG && !PMAPCHECK) */
1389: }
1390:
1391: struct pvo_entry *
1392: pmap_pvo_find_va(pmap_t pm, vaddr_t va, int *pteidx_p)
1393: {
1394: struct pvo_entry *pvo;
1395: int ptegidx;
1396:
1397: va &= ~ADDR_POFF;
1.2 matt 1398: ptegidx = va_to_pteg(pm, va);
1.1 matt 1399:
1400: TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
1401: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1402: if ((uintptr_t) pvo >= SEGMENT_LENGTH)
1403: panic("pmap_pvo_find_va: invalid pvo %p on "
1404: "list %#x (%p)", pvo, ptegidx,
1405: &pmap_pvo_table[ptegidx]);
1406: #endif
1407: if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
1408: if (pteidx_p)
1409: *pteidx_p = pmap_pvo_pte_index(pvo, ptegidx);
1410: return pvo;
1411: }
1412: }
1.38 sanjayl 1413: if ((pm == pmap_kernel()) && (va < SEGMENT_LENGTH))
1.54 mlelstv 1414: panic("%s: returning NULL for %s pmap, va: %#" _PRIxva "\n",
1.53 garbled 1415: __func__, (pm == pmap_kernel() ? "kernel" : "user"), va);
1.1 matt 1416: return NULL;
1417: }
1418:
1419: #if defined(DEBUG) || defined(PMAPCHECK)
1420: void
1421: pmap_pvo_check(const struct pvo_entry *pvo)
1422: {
1423: struct pvo_head *pvo_head;
1424: struct pvo_entry *pvo0;
1.2 matt 1425: volatile struct pte *pt;
1.1 matt 1426: int failed = 0;
1427:
1.50 ad 1428: PMAP_LOCK();
1429:
1.1 matt 1430: if ((uintptr_t)(pvo+1) >= SEGMENT_LENGTH)
1431: panic("pmap_pvo_check: pvo %p: invalid address", pvo);
1432:
1433: if ((uintptr_t)(pvo->pvo_pmap+1) >= SEGMENT_LENGTH) {
1434: printf("pmap_pvo_check: pvo %p: invalid pmap address %p\n",
1435: pvo, pvo->pvo_pmap);
1436: failed = 1;
1437: }
1438:
1439: if ((uintptr_t)TAILQ_NEXT(pvo, pvo_olink) >= SEGMENT_LENGTH ||
1440: (((uintptr_t)TAILQ_NEXT(pvo, pvo_olink)) & 0x1f) != 0) {
1441: printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n",
1442: pvo, TAILQ_NEXT(pvo, pvo_olink));
1443: failed = 1;
1444: }
1445:
1446: if ((uintptr_t)LIST_NEXT(pvo, pvo_vlink) >= SEGMENT_LENGTH ||
1447: (((uintptr_t)LIST_NEXT(pvo, pvo_vlink)) & 0x1f) != 0) {
1448: printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n",
1449: pvo, LIST_NEXT(pvo, pvo_vlink));
1450: failed = 1;
1451: }
1452:
1.39 matt 1453: if (PVO_MANAGED_P(pvo)) {
1.1 matt 1454: pvo_head = pa_to_pvoh(pvo->pvo_pte.pte_lo & PTE_RPGN, NULL);
1455: } else {
1456: if (pvo->pvo_vaddr < VM_MIN_KERNEL_ADDRESS) {
1457: printf("pmap_pvo_check: pvo %p: non kernel address "
1458: "on kernel unmanaged list\n", pvo);
1459: failed = 1;
1460: }
1461: pvo_head = &pmap_pvo_kunmanaged;
1462: }
1463: LIST_FOREACH(pvo0, pvo_head, pvo_vlink) {
1464: if (pvo0 == pvo)
1465: break;
1466: }
1467: if (pvo0 == NULL) {
1468: printf("pmap_pvo_check: pvo %p: not present "
1469: "on its vlist head %p\n", pvo, pvo_head);
1470: failed = 1;
1471: }
1472: if (pvo != pmap_pvo_find_va(pvo->pvo_pmap, pvo->pvo_vaddr, NULL)) {
1473: printf("pmap_pvo_check: pvo %p: not present "
1474: "on its olist head\n", pvo);
1475: failed = 1;
1476: }
1477: pt = pmap_pvo_to_pte(pvo, -1);
1478: if (pt == NULL) {
1479: if (pvo->pvo_pte.pte_hi & PTE_VALID) {
1480: printf("pmap_pvo_check: pvo %p: pte_hi VALID but "
1481: "no PTE\n", pvo);
1482: failed = 1;
1483: }
1484: } else {
1485: if ((uintptr_t) pt < (uintptr_t) &pmap_pteg_table[0] ||
1486: (uintptr_t) pt >=
1487: (uintptr_t) &pmap_pteg_table[pmap_pteg_cnt]) {
1488: printf("pmap_pvo_check: pvo %p: pte %p not in "
1489: "pteg table\n", pvo, pt);
1490: failed = 1;
1491: }
1492: if (((((uintptr_t) pt) >> 3) & 7) != PVO_PTEGIDX_GET(pvo)) {
1493: printf("pmap_pvo_check: pvo %p: pte_hi VALID but "
1494: "no PTE\n", pvo);
1495: failed = 1;
1496: }
1497: if (pvo->pvo_pte.pte_hi != pt->pte_hi) {
1498: printf("pmap_pvo_check: pvo %p: pte_hi differ: "
1.54 mlelstv 1499: "%#" _PRIxpte "/%#" _PRIxpte "\n", pvo,
1500: pvo->pvo_pte.pte_hi,
1501: pt->pte_hi);
1.1 matt 1502: failed = 1;
1503: }
1504: if (((pvo->pvo_pte.pte_lo ^ pt->pte_lo) &
1505: (PTE_PP|PTE_WIMG|PTE_RPGN)) != 0) {
1506: printf("pmap_pvo_check: pvo %p: pte_lo differ: "
1.54 mlelstv 1507: "%#" _PRIxpte "/%#" _PRIxpte "\n", pvo,
1508: (pvo->pvo_pte.pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN)),
1509: (pt->pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN)));
1.1 matt 1510: failed = 1;
1511: }
1512: if ((pmap_pte_to_va(pt) ^ PVO_VADDR(pvo)) & 0x0fffffff) {
1.53 garbled 1513: printf("pmap_pvo_check: pvo %p: PTE %p derived VA %#" _PRIxva ""
1514: " doesn't not match PVO's VA %#" _PRIxva "\n",
1.1 matt 1515: pvo, pt, pmap_pte_to_va(pt), PVO_VADDR(pvo));
1516: failed = 1;
1517: }
1518: if (failed)
1519: pmap_pte_print(pt);
1520: }
1521: if (failed)
1522: panic("pmap_pvo_check: pvo %p, pm %p: bugcheck!", pvo,
1523: pvo->pvo_pmap);
1.50 ad 1524:
1525: PMAP_UNLOCK();
1.1 matt 1526: }
1527: #endif /* DEBUG || PMAPCHECK */
1528:
1529: /*
1.25 chs 1530: * Search the PVO table looking for a non-wired entry.
1531: * If we find one, remove it and return it.
1532: */
1533:
1534: struct pvo_entry *
1535: pmap_pvo_reclaim(struct pmap *pm)
1536: {
1537: struct pvo_tqhead *pvoh;
1538: struct pvo_entry *pvo;
1539: uint32_t idx, endidx;
1540:
1541: endidx = pmap_pvo_reclaim_nextidx;
1542: for (idx = (endidx + 1) & pmap_pteg_mask; idx != endidx;
1543: idx = (idx + 1) & pmap_pteg_mask) {
1544: pvoh = &pmap_pvo_table[idx];
1545: TAILQ_FOREACH(pvo, pvoh, pvo_olink) {
1.39 matt 1546: if (!PVO_WIRED_P(pvo)) {
1.33 chs 1547: pmap_pvo_remove(pvo, -1, NULL);
1.25 chs 1548: pmap_pvo_reclaim_nextidx = idx;
1.26 matt 1549: PMAPCOUNT(pvos_reclaimed);
1.25 chs 1550: return pvo;
1551: }
1552: }
1553: }
1554: return NULL;
1555: }
1556:
1.95 chs 1557: static struct pool *
1558: pmap_pvo_pl(struct pvo_entry *pvo)
1559: {
1560:
1561: return PVO_MANAGED_P(pvo) ? &pmap_mpvo_pool : &pmap_upvo_pool;
1562: }
1563:
1.25 chs 1564: /*
1.1 matt 1565: * This returns whether this is the first mapping of a page.
1566: */
1567: int
1568: pmap_pvo_enter(pmap_t pm, struct pool *pl, struct pvo_head *pvo_head,
1.2 matt 1569: vaddr_t va, paddr_t pa, register_t pte_lo, int flags)
1.1 matt 1570: {
1571: struct pvo_entry *pvo;
1572: struct pvo_tqhead *pvoh;
1.2 matt 1573: register_t msr;
1.1 matt 1574: int ptegidx;
1575: int i;
1576: int poolflags = PR_NOWAIT;
1577:
1.28 chs 1578: /*
1579: * Compute the PTE Group index.
1580: */
1581: va &= ~ADDR_POFF;
1582: ptegidx = va_to_pteg(pm, va);
1583:
1584: msr = pmap_interrupts_off();
1585:
1.1 matt 1586: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1587: if (pmap_pvo_remove_depth > 0)
1588: panic("pmap_pvo_enter: called while pmap_pvo_remove active!");
1589: if (++pmap_pvo_enter_depth > 1)
1590: panic("pmap_pvo_enter: called recursively!");
1591: #endif
1592:
1593: /*
1594: * Remove any existing mapping for this page. Reuse the
1595: * pvo entry if there a mapping.
1596: */
1597: TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
1598: if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
1599: #ifdef DEBUG
1600: if ((pmapdebug & PMAPDEBUG_PVOENTER) &&
1601: ((pvo->pvo_pte.pte_lo ^ (pa|pte_lo)) &
1602: ~(PTE_REF|PTE_CHG)) == 0 &&
1603: va < VM_MIN_KERNEL_ADDRESS) {
1.56 phx 1604: printf("pmap_pvo_enter: pvo %p: dup %#" _PRIxpte "/%#" _PRIxpa "\n",
1.54 mlelstv 1605: pvo, pvo->pvo_pte.pte_lo, pte_lo|pa);
1.56 phx 1606: printf("pmap_pvo_enter: pte_hi=%#" _PRIxpte " sr=%#" _PRIsr "\n",
1.54 mlelstv 1607: pvo->pvo_pte.pte_hi,
1608: pm->pm_sr[va >> ADDR_SR_SHFT]);
1.1 matt 1609: pmap_pte_print(pmap_pvo_to_pte(pvo, -1));
1610: #ifdef DDBX
1611: Debugger();
1612: #endif
1613: }
1614: #endif
1615: PMAPCOUNT(mappings_replaced);
1.33 chs 1616: pmap_pvo_remove(pvo, -1, NULL);
1.1 matt 1617: break;
1618: }
1619: }
1620:
1621: /*
1622: * If we aren't overwriting an mapping, try to allocate
1623: */
1.26 matt 1624: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1625: --pmap_pvo_enter_depth;
1626: #endif
1.1 matt 1627: pmap_interrupts_restore(msr);
1.33 chs 1628: if (pvo) {
1.95 chs 1629: KASSERT(pmap_pvo_pl(pvo) == pl);
1630: } else {
1631: pvo = pool_get(pl, poolflags);
1.33 chs 1632: }
1.84 matt 1633: KASSERT((vaddr_t)pvo < VM_MIN_KERNEL_ADDRESS);
1.25 chs 1634:
1635: #ifdef DEBUG
1636: /*
1637: * Exercise pmap_pvo_reclaim() a little.
1638: */
1639: if (pvo && (flags & PMAP_CANFAIL) != 0 &&
1640: pmap_pvo_reclaim_debugctr++ > 0x1000 &&
1641: (pmap_pvo_reclaim_debugctr & 0xff) == 0) {
1642: pool_put(pl, pvo);
1643: pvo = NULL;
1644: }
1645: #endif
1646:
1.1 matt 1647: msr = pmap_interrupts_off();
1.26 matt 1648: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1649: ++pmap_pvo_enter_depth;
1650: #endif
1.1 matt 1651: if (pvo == NULL) {
1652: pvo = pmap_pvo_reclaim(pm);
1653: if (pvo == NULL) {
1654: if ((flags & PMAP_CANFAIL) == 0)
1655: panic("pmap_pvo_enter: failed");
1656: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1657: pmap_pvo_enter_depth--;
1658: #endif
1.26 matt 1659: PMAPCOUNT(pvos_failed);
1.1 matt 1660: pmap_interrupts_restore(msr);
1661: return ENOMEM;
1662: }
1663: }
1.25 chs 1664:
1.1 matt 1665: pvo->pvo_vaddr = va;
1666: pvo->pvo_pmap = pm;
1667: pvo->pvo_vaddr &= ~ADDR_POFF;
1668: if (flags & VM_PROT_EXECUTE) {
1669: PMAPCOUNT(exec_mappings);
1.14 chs 1670: pvo_set_exec(pvo);
1.1 matt 1671: }
1672: if (flags & PMAP_WIRED)
1673: pvo->pvo_vaddr |= PVO_WIRED;
1674: if (pvo_head != &pmap_pvo_kunmanaged) {
1675: pvo->pvo_vaddr |= PVO_MANAGED;
1676: PMAPCOUNT(mappings);
1677: } else {
1678: PMAPCOUNT(kernel_mappings);
1679: }
1.2 matt 1680: pmap_pte_create(&pvo->pvo_pte, pm, va, pa | pte_lo);
1.1 matt 1681:
1682: LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink);
1.39 matt 1683: if (PVO_WIRED_P(pvo))
1.1 matt 1684: pvo->pvo_pmap->pm_stats.wired_count++;
1685: pvo->pvo_pmap->pm_stats.resident_count++;
1686: #if defined(DEBUG)
1.38 sanjayl 1687: /* if (pm != pmap_kernel() && va < VM_MIN_KERNEL_ADDRESS) */
1.1 matt 1688: DPRINTFN(PVOENTER,
1.85 matt 1689: "pmap_pvo_enter: pvo %p: pm %p va %#" _PRIxva " pa %#" _PRIxpa "\n",
1690: pvo, pm, va, pa);
1.1 matt 1691: #endif
1692:
1693: /*
1694: * We hope this succeeds but it isn't required.
1695: */
1696: pvoh = &pmap_pvo_table[ptegidx];
1697: i = pmap_pte_insert(ptegidx, &pvo->pvo_pte);
1698: if (i >= 0) {
1699: PVO_PTEGIDX_SET(pvo, i);
1.12 matt 1700: PVO_WHERE(pvo, ENTER_INSERT);
1.1 matt 1701: PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID)
1702: ? pmap_evcnt_ptes_secondary : pmap_evcnt_ptes_primary)[i]);
1703: TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink);
1.38 sanjayl 1704:
1.1 matt 1705: } else {
1706: /*
1707: * Since we didn't have room for this entry (which makes it
1708: * and evicted entry), place it at the head of the list.
1709: */
1710: TAILQ_INSERT_HEAD(pvoh, pvo, pvo_olink);
1711: PMAPCOUNT(ptes_evicted);
1712: pm->pm_evictions++;
1.12 matt 1713: /*
1714: * If this is a kernel page, make sure it's active.
1715: */
1716: if (pm == pmap_kernel()) {
1.45 thorpej 1717: i = pmap_pte_spill(pm, va, false);
1.12 matt 1718: KASSERT(i);
1719: }
1.1 matt 1720: }
1721: PMAP_PVO_CHECK(pvo); /* sanity check */
1722: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1723: pmap_pvo_enter_depth--;
1724: #endif
1725: pmap_interrupts_restore(msr);
1726: return 0;
1727: }
1728:
1.53 garbled 1729: static void
1.33 chs 1730: pmap_pvo_remove(struct pvo_entry *pvo, int pteidx, struct pvo_head *pvol)
1.1 matt 1731: {
1.2 matt 1732: volatile struct pte *pt;
1.1 matt 1733: int ptegidx;
1734:
1735: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1736: if (++pmap_pvo_remove_depth > 1)
1737: panic("pmap_pvo_remove: called recursively!");
1738: #endif
1739:
1740: /*
1741: * If we haven't been supplied the ptegidx, calculate it.
1742: */
1743: if (pteidx == -1) {
1.2 matt 1744: ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr);
1.1 matt 1745: pteidx = pmap_pvo_pte_index(pvo, ptegidx);
1746: } else {
1747: ptegidx = pteidx >> 3;
1748: if (pvo->pvo_pte.pte_hi & PTE_HID)
1749: ptegidx ^= pmap_pteg_mask;
1750: }
1751: PMAP_PVO_CHECK(pvo); /* sanity check */
1752:
1753: /*
1754: * If there is an active pte entry, we need to deactivate it
1755: * (and save the ref & chg bits).
1756: */
1757: pt = pmap_pvo_to_pte(pvo, pteidx);
1758: if (pt != NULL) {
1759: pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
1.12 matt 1760: PVO_WHERE(pvo, REMOVE);
1.1 matt 1761: PVO_PTEGIDX_CLR(pvo);
1762: PMAPCOUNT(ptes_removed);
1763: } else {
1764: KASSERT(pvo->pvo_pmap->pm_evictions > 0);
1765: pvo->pvo_pmap->pm_evictions--;
1766: }
1767:
1768: /*
1.14 chs 1769: * Account for executable mappings.
1770: */
1.39 matt 1771: if (PVO_EXECUTABLE_P(pvo))
1.14 chs 1772: pvo_clear_exec(pvo);
1773:
1774: /*
1775: * Update our statistics.
1.1 matt 1776: */
1777: pvo->pvo_pmap->pm_stats.resident_count--;
1.39 matt 1778: if (PVO_WIRED_P(pvo))
1.1 matt 1779: pvo->pvo_pmap->pm_stats.wired_count--;
1780:
1781: /*
1782: * Save the REF/CHG bits into their cache if the page is managed.
1783: */
1.39 matt 1784: if (PVO_MANAGED_P(pvo)) {
1.2 matt 1785: register_t ptelo = pvo->pvo_pte.pte_lo;
1.1 matt 1786: struct vm_page *pg = PHYS_TO_VM_PAGE(ptelo & PTE_RPGN);
1787:
1788: if (pg != NULL) {
1.37 matt 1789: /*
1790: * If this page was changed and it is mapped exec,
1791: * invalidate it.
1792: */
1793: if ((ptelo & PTE_CHG) &&
1794: (pmap_attr_fetch(pg) & PTE_EXEC)) {
1795: struct pvo_head *pvoh = vm_page_to_pvoh(pg);
1796: if (LIST_EMPTY(pvoh)) {
1.85 matt 1797: DPRINTFN(EXEC, "[pmap_pvo_remove: "
1.53 garbled 1798: "%#" _PRIxpa ": clear-exec]\n",
1.85 matt 1799: VM_PAGE_TO_PHYS(pg));
1.37 matt 1800: pmap_attr_clear(pg, PTE_EXEC);
1801: PMAPCOUNT(exec_uncached_pvo_remove);
1802: } else {
1.85 matt 1803: DPRINTFN(EXEC, "[pmap_pvo_remove: "
1.53 garbled 1804: "%#" _PRIxpa ": syncicache]\n",
1.85 matt 1805: VM_PAGE_TO_PHYS(pg));
1.37 matt 1806: pmap_syncicache(VM_PAGE_TO_PHYS(pg),
1807: PAGE_SIZE);
1808: PMAPCOUNT(exec_synced_pvo_remove);
1809: }
1810: }
1811:
1.1 matt 1812: pmap_attr_save(pg, ptelo & (PTE_REF|PTE_CHG));
1813: }
1814: PMAPCOUNT(unmappings);
1815: } else {
1816: PMAPCOUNT(kernel_unmappings);
1817: }
1818:
1819: /*
1820: * Remove the PVO from its lists and return it to the pool.
1821: */
1822: LIST_REMOVE(pvo, pvo_vlink);
1823: TAILQ_REMOVE(&pmap_pvo_table[ptegidx], pvo, pvo_olink);
1.33 chs 1824: if (pvol) {
1825: LIST_INSERT_HEAD(pvol, pvo, pvo_vlink);
1.25 chs 1826: }
1.1 matt 1827: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1828: pmap_pvo_remove_depth--;
1829: #endif
1830: }
1831:
1.33 chs 1832: void
1833: pmap_pvo_free(struct pvo_entry *pvo)
1834: {
1835:
1.95 chs 1836: pool_put(pmap_pvo_pl(pvo), pvo);
1.33 chs 1837: }
1838:
1839: void
1840: pmap_pvo_free_list(struct pvo_head *pvol)
1841: {
1842: struct pvo_entry *pvo, *npvo;
1843:
1844: for (pvo = LIST_FIRST(pvol); pvo != NULL; pvo = npvo) {
1845: npvo = LIST_NEXT(pvo, pvo_vlink);
1846: LIST_REMOVE(pvo, pvo_vlink);
1847: pmap_pvo_free(pvo);
1848: }
1849: }
1850:
1.1 matt 1851: /*
1.14 chs 1852: * Mark a mapping as executable.
1853: * If this is the first executable mapping in the segment,
1854: * clear the noexec flag.
1855: */
1.53 garbled 1856: static void
1.14 chs 1857: pvo_set_exec(struct pvo_entry *pvo)
1858: {
1859: struct pmap *pm = pvo->pvo_pmap;
1860:
1.39 matt 1861: if (pm == pmap_kernel() || PVO_EXECUTABLE_P(pvo)) {
1.14 chs 1862: return;
1863: }
1864: pvo->pvo_vaddr |= PVO_EXECUTABLE;
1.53 garbled 1865: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.18 matt 1866: {
1867: int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT;
1868: if (pm->pm_exec[sr]++ == 0) {
1869: pm->pm_sr[sr] &= ~SR_NOEXEC;
1870: }
1.14 chs 1871: }
1.18 matt 1872: #endif
1.14 chs 1873: }
1874:
1875: /*
1876: * Mark a mapping as non-executable.
1877: * If this was the last executable mapping in the segment,
1878: * set the noexec flag.
1879: */
1.53 garbled 1880: static void
1.14 chs 1881: pvo_clear_exec(struct pvo_entry *pvo)
1882: {
1883: struct pmap *pm = pvo->pvo_pmap;
1884:
1.39 matt 1885: if (pm == pmap_kernel() || !PVO_EXECUTABLE_P(pvo)) {
1.14 chs 1886: return;
1887: }
1888: pvo->pvo_vaddr &= ~PVO_EXECUTABLE;
1.53 garbled 1889: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.18 matt 1890: {
1891: int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT;
1892: if (--pm->pm_exec[sr] == 0) {
1893: pm->pm_sr[sr] |= SR_NOEXEC;
1894: }
1.14 chs 1895: }
1.18 matt 1896: #endif
1.14 chs 1897: }
1898:
1899: /*
1.1 matt 1900: * Insert physical page at pa into the given pmap at virtual address va.
1901: */
1902: int
1.65 cegger 1903: pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1.1 matt 1904: {
1905: struct mem_region *mp;
1906: struct pvo_head *pvo_head;
1907: struct vm_page *pg;
1908: struct pool *pl;
1.2 matt 1909: register_t pte_lo;
1.1 matt 1910: int error;
1911: u_int was_exec = 0;
1912:
1.50 ad 1913: PMAP_LOCK();
1914:
1.1 matt 1915: if (__predict_false(!pmap_initialized)) {
1916: pvo_head = &pmap_pvo_kunmanaged;
1917: pl = &pmap_upvo_pool;
1918: pg = NULL;
1919: was_exec = PTE_EXEC;
1920: } else {
1921: pvo_head = pa_to_pvoh(pa, &pg);
1922: pl = &pmap_mpvo_pool;
1923: }
1924:
1925: DPRINTFN(ENTER,
1.85 matt 1926: "pmap_enter(%p, %#" _PRIxva ", %#" _PRIxpa ", 0x%x, 0x%x):",
1927: pm, va, pa, prot, flags);
1.1 matt 1928:
1929: /*
1930: * If this is a managed page, and it's the first reference to the
1931: * page clear the execness of the page. Otherwise fetch the execness.
1932: */
1933: if (pg != NULL)
1934: was_exec = pmap_attr_fetch(pg) & PTE_EXEC;
1935:
1.85 matt 1936: DPRINTFN(ENTER, " was_exec=%d", was_exec);
1.1 matt 1937:
1938: /*
1939: * Assume the page is cache inhibited and access is guarded unless
1940: * it's in our available memory array. If it is in the memory array,
1941: * asssume it's in memory coherent memory.
1942: */
1.77 macallan 1943: if (flags & PMAP_MD_PREFETCHABLE) {
1944: pte_lo = 0;
1945: } else
1946: pte_lo = PTE_G;
1947:
1.81 matt 1948: if ((flags & PMAP_NOCACHE) == 0) {
1.1 matt 1949: for (mp = mem; mp->size; mp++) {
1950: if (pa >= mp->start && pa < mp->start + mp->size) {
1951: pte_lo = PTE_M;
1952: break;
1953: }
1954: }
1.87 kiyohara 1955: #ifdef MULTIPROCESSOR
1956: if (((mfpvr() >> 16) & 0xffff) == MPC603e)
1957: pte_lo = PTE_M;
1958: #endif
1.77 macallan 1959: } else {
1960: pte_lo |= PTE_I;
1.1 matt 1961: }
1962:
1963: if (prot & VM_PROT_WRITE)
1964: pte_lo |= PTE_BW;
1965: else
1966: pte_lo |= PTE_BR;
1967:
1968: /*
1969: * If this was in response to a fault, "pre-fault" the PTE's
1970: * changed/referenced bit appropriately.
1971: */
1972: if (flags & VM_PROT_WRITE)
1973: pte_lo |= PTE_CHG;
1.30 chs 1974: if (flags & VM_PROT_ALL)
1.1 matt 1975: pte_lo |= PTE_REF;
1976:
1977: /*
1978: * We need to know if this page can be executable
1979: */
1980: flags |= (prot & VM_PROT_EXECUTE);
1981:
1982: /*
1983: * Record mapping for later back-translation and pte spilling.
1984: * This will overwrite any existing mapping.
1985: */
1986: error = pmap_pvo_enter(pm, pl, pvo_head, va, pa, pte_lo, flags);
1987:
1988: /*
1989: * Flush the real page from the instruction cache if this page is
1990: * mapped executable and cacheable and has not been flushed since
1991: * the last time it was modified.
1992: */
1993: if (error == 0 &&
1994: (flags & VM_PROT_EXECUTE) &&
1995: (pte_lo & PTE_I) == 0 &&
1996: was_exec == 0) {
1.85 matt 1997: DPRINTFN(ENTER, " %s", "syncicache");
1.1 matt 1998: PMAPCOUNT(exec_synced);
1.6 thorpej 1999: pmap_syncicache(pa, PAGE_SIZE);
1.1 matt 2000: if (pg != NULL) {
2001: pmap_attr_save(pg, PTE_EXEC);
2002: PMAPCOUNT(exec_cached);
2003: #if defined(DEBUG) || defined(PMAPDEBUG)
2004: if (pmapdebug & PMAPDEBUG_ENTER)
2005: printf(" marked-as-exec");
2006: else if (pmapdebug & PMAPDEBUG_EXEC)
1.53 garbled 2007: printf("[pmap_enter: %#" _PRIxpa ": marked-as-exec]\n",
1.34 yamt 2008: VM_PAGE_TO_PHYS(pg));
1.1 matt 2009:
2010: #endif
2011: }
2012: }
2013:
1.85 matt 2014: DPRINTFN(ENTER, ": error=%d\n", error);
1.1 matt 2015:
1.50 ad 2016: PMAP_UNLOCK();
2017:
1.1 matt 2018: return error;
2019: }
2020:
2021: void
1.68 cegger 2022: pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1.1 matt 2023: {
2024: struct mem_region *mp;
1.2 matt 2025: register_t pte_lo;
1.1 matt 2026: int error;
2027:
1.85 matt 2028: #if defined (PMAP_OEA64_BRIDGE) || defined (PMAP_OEA)
1.1 matt 2029: if (va < VM_MIN_KERNEL_ADDRESS)
2030: panic("pmap_kenter_pa: attempt to enter "
1.53 garbled 2031: "non-kernel address %#" _PRIxva "!", va);
1.38 sanjayl 2032: #endif
1.1 matt 2033:
2034: DPRINTFN(KENTER,
1.85 matt 2035: "pmap_kenter_pa(%#" _PRIxva ",%#" _PRIxpa ",%#x)\n", va, pa, prot);
1.1 matt 2036:
1.50 ad 2037: PMAP_LOCK();
2038:
1.1 matt 2039: /*
2040: * Assume the page is cache inhibited and access is guarded unless
2041: * it's in our available memory array. If it is in the memory array,
2042: * asssume it's in memory coherent memory.
2043: */
2044: pte_lo = PTE_IG;
1.81 matt 2045: if ((flags & PMAP_NOCACHE) == 0) {
1.4 matt 2046: for (mp = mem; mp->size; mp++) {
2047: if (pa >= mp->start && pa < mp->start + mp->size) {
2048: pte_lo = PTE_M;
2049: break;
2050: }
1.1 matt 2051: }
1.87 kiyohara 2052: #ifdef MULTIPROCESSOR
2053: if (((mfpvr() >> 16) & 0xffff) == MPC603e)
2054: pte_lo = PTE_M;
2055: #endif
1.1 matt 2056: }
2057:
2058: if (prot & VM_PROT_WRITE)
2059: pte_lo |= PTE_BW;
2060: else
2061: pte_lo |= PTE_BR;
2062:
2063: /*
2064: * We don't care about REF/CHG on PVOs on the unmanaged list.
2065: */
2066: error = pmap_pvo_enter(pmap_kernel(), &pmap_upvo_pool,
2067: &pmap_pvo_kunmanaged, va, pa, pte_lo, prot|PMAP_WIRED);
2068:
2069: if (error != 0)
1.53 garbled 2070: panic("pmap_kenter_pa: failed to enter va %#" _PRIxva " pa %#" _PRIxpa ": %d",
1.1 matt 2071: va, pa, error);
1.50 ad 2072:
2073: PMAP_UNLOCK();
1.1 matt 2074: }
2075:
2076: void
2077: pmap_kremove(vaddr_t va, vsize_t len)
2078: {
2079: if (va < VM_MIN_KERNEL_ADDRESS)
2080: panic("pmap_kremove: attempt to remove "
1.53 garbled 2081: "non-kernel address %#" _PRIxva "!", va);
1.1 matt 2082:
1.85 matt 2083: DPRINTFN(KREMOVE, "pmap_kremove(%#" _PRIxva ",%#" _PRIxva ")\n", va, len);
1.1 matt 2084: pmap_remove(pmap_kernel(), va, va + len);
2085: }
2086:
2087: /*
2088: * Remove the given range of mapping entries.
2089: */
2090: void
2091: pmap_remove(pmap_t pm, vaddr_t va, vaddr_t endva)
2092: {
1.33 chs 2093: struct pvo_head pvol;
1.1 matt 2094: struct pvo_entry *pvo;
1.2 matt 2095: register_t msr;
1.1 matt 2096: int pteidx;
2097:
1.50 ad 2098: PMAP_LOCK();
1.33 chs 2099: LIST_INIT(&pvol);
1.14 chs 2100: msr = pmap_interrupts_off();
1.1 matt 2101: for (; va < endva; va += PAGE_SIZE) {
2102: pvo = pmap_pvo_find_va(pm, va, &pteidx);
2103: if (pvo != NULL) {
1.33 chs 2104: pmap_pvo_remove(pvo, pteidx, &pvol);
1.1 matt 2105: }
2106: }
1.14 chs 2107: pmap_interrupts_restore(msr);
1.33 chs 2108: pmap_pvo_free_list(&pvol);
1.50 ad 2109: PMAP_UNLOCK();
1.1 matt 2110: }
2111:
2112: /*
2113: * Get the physical page address for the given pmap/virtual address.
2114: */
1.44 thorpej 2115: bool
1.1 matt 2116: pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap)
2117: {
2118: struct pvo_entry *pvo;
1.2 matt 2119: register_t msr;
1.7 matt 2120:
1.50 ad 2121: PMAP_LOCK();
1.38 sanjayl 2122:
1.7 matt 2123: /*
2124: * If this is a kernel pmap lookup, also check the battable
2125: * and if we get a hit, translate the VA to a PA using the
1.36 nathanw 2126: * BAT entries. Don't check for VM_MAX_KERNEL_ADDRESS is
1.7 matt 2127: * that will wrap back to 0.
2128: */
2129: if (pm == pmap_kernel() &&
2130: (va < VM_MIN_KERNEL_ADDRESS ||
2131: (KERNEL2_SR < 15 && VM_MAX_KERNEL_ADDRESS <= va))) {
1.8 matt 2132: KASSERT((va >> ADDR_SR_SHFT) != USER_SR);
1.53 garbled 2133: #if defined (PMAP_OEA)
1.55 garbled 2134: #ifdef PPC_OEA601
2135: if ((MFPVR() >> 16) == MPC601) {
1.24 kleink 2136: register_t batu = battable[va >> 23].batu;
2137: register_t batl = battable[va >> 23].batl;
2138: register_t sr = iosrtable[va >> ADDR_SR_SHFT];
2139: if (BAT601_VALID_P(batl) &&
2140: BAT601_VA_MATCH_P(batu, batl, va)) {
2141: register_t mask =
2142: (~(batl & BAT601_BSM) << 17) & ~0x1ffffL;
1.29 briggs 2143: if (pap)
2144: *pap = (batl & mask) | (va & ~mask);
1.50 ad 2145: PMAP_UNLOCK();
1.45 thorpej 2146: return true;
1.24 kleink 2147: } else if (SR601_VALID_P(sr) &&
2148: SR601_PA_MATCH_P(sr, va)) {
1.29 briggs 2149: if (pap)
2150: *pap = va;
1.50 ad 2151: PMAP_UNLOCK();
1.45 thorpej 2152: return true;
1.24 kleink 2153: }
1.55 garbled 2154: } else
2155: #endif /* PPC_OEA601 */
2156: {
1.83 matt 2157: register_t batu = battable[BAT_VA2IDX(va)].batu;
1.55 garbled 2158: if (BAT_VALID_P(batu,0) && BAT_VA_MATCH_P(batu,va)) {
1.83 matt 2159: register_t batl = battable[BAT_VA2IDX(va)].batl;
1.55 garbled 2160: register_t mask =
1.83 matt 2161: (~(batu & (BAT_XBL|BAT_BL)) << 15) & ~0x1ffffL;
1.55 garbled 2162: if (pap)
2163: *pap = (batl & mask) | (va & ~mask);
2164: PMAP_UNLOCK();
2165: return true;
2166: }
1.7 matt 2167: }
1.96 rin 2168: PMAP_UNLOCK();
1.45 thorpej 2169: return false;
1.53 garbled 2170: #elif defined (PMAP_OEA64_BRIDGE)
1.52 garbled 2171: if (va >= SEGMENT_LENGTH)
2172: panic("%s: pm: %s va >= SEGMENT_LENGTH, va: 0x%08lx\n",
2173: __func__, (pm == pmap_kernel() ? "kernel" : "user"), va);
2174: else {
2175: if (pap)
2176: *pap = va;
2177: PMAP_UNLOCK();
2178: return true;
2179: }
1.53 garbled 2180: #elif defined (PMAP_OEA64)
1.38 sanjayl 2181: #error PPC_OEA64 not supported
2182: #endif /* PPC_OEA */
1.7 matt 2183: }
1.1 matt 2184:
2185: msr = pmap_interrupts_off();
2186: pvo = pmap_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
2187: if (pvo != NULL) {
2188: PMAP_PVO_CHECK(pvo); /* sanity check */
1.29 briggs 2189: if (pap)
2190: *pap = (pvo->pvo_pte.pte_lo & PTE_RPGN)
2191: | (va & ADDR_POFF);
1.1 matt 2192: }
2193: pmap_interrupts_restore(msr);
1.50 ad 2194: PMAP_UNLOCK();
1.1 matt 2195: return pvo != NULL;
2196: }
2197:
2198: /*
2199: * Lower the protection on the specified range of this pmap.
2200: */
2201: void
2202: pmap_protect(pmap_t pm, vaddr_t va, vaddr_t endva, vm_prot_t prot)
2203: {
2204: struct pvo_entry *pvo;
1.2 matt 2205: volatile struct pte *pt;
2206: register_t msr;
1.1 matt 2207: int pteidx;
2208:
2209: /*
2210: * Since this routine only downgrades protection, we should
1.14 chs 2211: * always be called with at least one bit not set.
1.1 matt 2212: */
1.14 chs 2213: KASSERT(prot != VM_PROT_ALL);
1.1 matt 2214:
2215: /*
2216: * If there is no protection, this is equivalent to
2217: * remove the pmap from the pmap.
2218: */
2219: if ((prot & VM_PROT_READ) == 0) {
2220: pmap_remove(pm, va, endva);
2221: return;
2222: }
2223:
1.50 ad 2224: PMAP_LOCK();
2225:
1.1 matt 2226: msr = pmap_interrupts_off();
1.6 thorpej 2227: for (; va < endva; va += PAGE_SIZE) {
1.1 matt 2228: pvo = pmap_pvo_find_va(pm, va, &pteidx);
2229: if (pvo == NULL)
2230: continue;
2231: PMAP_PVO_CHECK(pvo); /* sanity check */
2232:
2233: /*
2234: * Revoke executable if asked to do so.
2235: */
2236: if ((prot & VM_PROT_EXECUTE) == 0)
1.14 chs 2237: pvo_clear_exec(pvo);
1.1 matt 2238:
2239: #if 0
2240: /*
2241: * If the page is already read-only, no change
2242: * needs to be made.
2243: */
2244: if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR)
2245: continue;
2246: #endif
2247: /*
2248: * Grab the PTE pointer before we diddle with
2249: * the cached PTE copy.
2250: */
2251: pt = pmap_pvo_to_pte(pvo, pteidx);
2252: /*
2253: * Change the protection of the page.
2254: */
2255: pvo->pvo_pte.pte_lo &= ~PTE_PP;
2256: pvo->pvo_pte.pte_lo |= PTE_BR;
2257:
2258: /*
2259: * If the PVO is in the page table, update
2260: * that pte at well.
2261: */
2262: if (pt != NULL) {
2263: pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
1.12 matt 2264: PVO_WHERE(pvo, PMAP_PROTECT);
1.1 matt 2265: PMAPCOUNT(ptes_changed);
2266: }
2267:
2268: PMAP_PVO_CHECK(pvo); /* sanity check */
2269: }
2270: pmap_interrupts_restore(msr);
1.50 ad 2271: PMAP_UNLOCK();
1.1 matt 2272: }
2273:
2274: void
2275: pmap_unwire(pmap_t pm, vaddr_t va)
2276: {
2277: struct pvo_entry *pvo;
1.2 matt 2278: register_t msr;
1.1 matt 2279:
1.50 ad 2280: PMAP_LOCK();
1.1 matt 2281: msr = pmap_interrupts_off();
2282: pvo = pmap_pvo_find_va(pm, va, NULL);
2283: if (pvo != NULL) {
1.39 matt 2284: if (PVO_WIRED_P(pvo)) {
1.1 matt 2285: pvo->pvo_vaddr &= ~PVO_WIRED;
2286: pm->pm_stats.wired_count--;
2287: }
2288: PMAP_PVO_CHECK(pvo); /* sanity check */
2289: }
2290: pmap_interrupts_restore(msr);
1.50 ad 2291: PMAP_UNLOCK();
1.1 matt 2292: }
2293:
2294: /*
2295: * Lower the protection on the specified physical page.
2296: */
2297: void
2298: pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
2299: {
1.33 chs 2300: struct pvo_head *pvo_head, pvol;
1.1 matt 2301: struct pvo_entry *pvo, *next_pvo;
1.2 matt 2302: volatile struct pte *pt;
2303: register_t msr;
1.1 matt 2304:
1.50 ad 2305: PMAP_LOCK();
2306:
1.14 chs 2307: KASSERT(prot != VM_PROT_ALL);
1.33 chs 2308: LIST_INIT(&pvol);
1.1 matt 2309: msr = pmap_interrupts_off();
2310:
2311: /*
2312: * When UVM reuses a page, it does a pmap_page_protect with
2313: * VM_PROT_NONE. At that point, we can clear the exec flag
2314: * since we know the page will have different contents.
2315: */
2316: if ((prot & VM_PROT_READ) == 0) {
1.85 matt 2317: DPRINTFN(EXEC, "[pmap_page_protect: %#" _PRIxpa ": clear-exec]\n",
2318: VM_PAGE_TO_PHYS(pg));
1.1 matt 2319: if (pmap_attr_fetch(pg) & PTE_EXEC) {
2320: PMAPCOUNT(exec_uncached_page_protect);
2321: pmap_attr_clear(pg, PTE_EXEC);
2322: }
2323: }
2324:
2325: pvo_head = vm_page_to_pvoh(pg);
2326: for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) {
2327: next_pvo = LIST_NEXT(pvo, pvo_vlink);
2328: PMAP_PVO_CHECK(pvo); /* sanity check */
2329:
2330: /*
2331: * Downgrading to no mapping at all, we just remove the entry.
2332: */
2333: if ((prot & VM_PROT_READ) == 0) {
1.33 chs 2334: pmap_pvo_remove(pvo, -1, &pvol);
1.1 matt 2335: continue;
2336: }
2337:
2338: /*
2339: * If EXEC permission is being revoked, just clear the
2340: * flag in the PVO.
2341: */
2342: if ((prot & VM_PROT_EXECUTE) == 0)
1.14 chs 2343: pvo_clear_exec(pvo);
1.1 matt 2344:
2345: /*
2346: * If this entry is already RO, don't diddle with the
2347: * page table.
2348: */
2349: if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) {
2350: PMAP_PVO_CHECK(pvo);
2351: continue;
2352: }
2353:
2354: /*
2355: * Grab the PTE before the we diddle the bits so
2356: * pvo_to_pte can verify the pte contents are as
2357: * expected.
2358: */
2359: pt = pmap_pvo_to_pte(pvo, -1);
2360: pvo->pvo_pte.pte_lo &= ~PTE_PP;
2361: pvo->pvo_pte.pte_lo |= PTE_BR;
2362: if (pt != NULL) {
2363: pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
1.12 matt 2364: PVO_WHERE(pvo, PMAP_PAGE_PROTECT);
1.1 matt 2365: PMAPCOUNT(ptes_changed);
2366: }
2367: PMAP_PVO_CHECK(pvo); /* sanity check */
2368: }
2369: pmap_interrupts_restore(msr);
1.33 chs 2370: pmap_pvo_free_list(&pvol);
1.50 ad 2371:
2372: PMAP_UNLOCK();
1.1 matt 2373: }
2374:
2375: /*
2376: * Activate the address space for the specified process. If the process
2377: * is the current process, load the new MMU context.
2378: */
2379: void
2380: pmap_activate(struct lwp *l)
2381: {
1.69 rmind 2382: struct pcb *pcb = lwp_getpcb(l);
1.1 matt 2383: pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap;
2384:
2385: DPRINTFN(ACTIVATE,
1.85 matt 2386: "pmap_activate: lwp %p (curlwp %p)\n", l, curlwp);
1.1 matt 2387:
2388: /*
1.70 skrll 2389: * XXX Normally performed in cpu_lwp_fork().
1.1 matt 2390: */
1.13 matt 2391: pcb->pcb_pm = pmap;
1.17 matt 2392:
2393: /*
2394: * In theory, the SR registers need only be valid on return
2395: * to user space wait to do them there.
2396: */
2397: if (l == curlwp) {
2398: /* Store pointer to new current pmap. */
2399: curpm = pmap;
2400: }
1.1 matt 2401: }
2402:
2403: /*
2404: * Deactivate the specified process's address space.
2405: */
2406: void
2407: pmap_deactivate(struct lwp *l)
2408: {
2409: }
2410:
1.44 thorpej 2411: bool
1.1 matt 2412: pmap_query_bit(struct vm_page *pg, int ptebit)
2413: {
2414: struct pvo_entry *pvo;
1.2 matt 2415: volatile struct pte *pt;
2416: register_t msr;
1.1 matt 2417:
1.50 ad 2418: PMAP_LOCK();
2419:
2420: if (pmap_attr_fetch(pg) & ptebit) {
2421: PMAP_UNLOCK();
1.45 thorpej 2422: return true;
1.50 ad 2423: }
1.14 chs 2424:
1.1 matt 2425: msr = pmap_interrupts_off();
2426: LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) {
2427: PMAP_PVO_CHECK(pvo); /* sanity check */
2428: /*
2429: * See if we saved the bit off. If so cache, it and return
2430: * success.
2431: */
2432: if (pvo->pvo_pte.pte_lo & ptebit) {
2433: pmap_attr_save(pg, ptebit);
2434: PMAP_PVO_CHECK(pvo); /* sanity check */
2435: pmap_interrupts_restore(msr);
1.50 ad 2436: PMAP_UNLOCK();
1.45 thorpej 2437: return true;
1.1 matt 2438: }
2439: }
2440: /*
2441: * No luck, now go thru the hard part of looking at the ptes
2442: * themselves. Sync so any pending REF/CHG bits are flushed
2443: * to the PTEs.
2444: */
2445: SYNC();
2446: LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) {
2447: PMAP_PVO_CHECK(pvo); /* sanity check */
2448: /*
2449: * See if this pvo have a valid PTE. If so, fetch the
2450: * REF/CHG bits from the valid PTE. If the appropriate
2451: * ptebit is set, cache, it and return success.
2452: */
2453: pt = pmap_pvo_to_pte(pvo, -1);
2454: if (pt != NULL) {
2455: pmap_pte_synch(pt, &pvo->pvo_pte);
2456: if (pvo->pvo_pte.pte_lo & ptebit) {
2457: pmap_attr_save(pg, ptebit);
2458: PMAP_PVO_CHECK(pvo); /* sanity check */
2459: pmap_interrupts_restore(msr);
1.50 ad 2460: PMAP_UNLOCK();
1.45 thorpej 2461: return true;
1.1 matt 2462: }
2463: }
2464: }
2465: pmap_interrupts_restore(msr);
1.50 ad 2466: PMAP_UNLOCK();
1.45 thorpej 2467: return false;
1.1 matt 2468: }
2469:
1.44 thorpej 2470: bool
1.1 matt 2471: pmap_clear_bit(struct vm_page *pg, int ptebit)
2472: {
2473: struct pvo_head *pvoh = vm_page_to_pvoh(pg);
2474: struct pvo_entry *pvo;
1.2 matt 2475: volatile struct pte *pt;
2476: register_t msr;
1.1 matt 2477: int rv = 0;
2478:
1.50 ad 2479: PMAP_LOCK();
1.1 matt 2480: msr = pmap_interrupts_off();
2481:
2482: /*
2483: * Fetch the cache value
2484: */
2485: rv |= pmap_attr_fetch(pg);
2486:
2487: /*
2488: * Clear the cached value.
2489: */
2490: pmap_attr_clear(pg, ptebit);
2491:
2492: /*
2493: * Sync so any pending REF/CHG bits are flushed to the PTEs (so we
2494: * can reset the right ones). Note that since the pvo entries and
2495: * list heads are accessed via BAT0 and are never placed in the
2496: * page table, we don't have to worry about further accesses setting
2497: * the REF/CHG bits.
2498: */
2499: SYNC();
2500:
2501: /*
2502: * For each pvo entry, clear pvo's ptebit. If this pvo have a
2503: * valid PTE. If so, clear the ptebit from the valid PTE.
2504: */
2505: LIST_FOREACH(pvo, pvoh, pvo_vlink) {
2506: PMAP_PVO_CHECK(pvo); /* sanity check */
2507: pt = pmap_pvo_to_pte(pvo, -1);
2508: if (pt != NULL) {
2509: /*
2510: * Only sync the PTE if the bit we are looking
2511: * for is not already set.
2512: */
2513: if ((pvo->pvo_pte.pte_lo & ptebit) == 0)
2514: pmap_pte_synch(pt, &pvo->pvo_pte);
2515: /*
2516: * If the bit we are looking for was already set,
2517: * clear that bit in the pte.
2518: */
2519: if (pvo->pvo_pte.pte_lo & ptebit)
2520: pmap_pte_clear(pt, PVO_VADDR(pvo), ptebit);
2521: }
2522: rv |= pvo->pvo_pte.pte_lo & (PTE_CHG|PTE_REF);
2523: pvo->pvo_pte.pte_lo &= ~ptebit;
2524: PMAP_PVO_CHECK(pvo); /* sanity check */
2525: }
2526: pmap_interrupts_restore(msr);
1.14 chs 2527:
1.1 matt 2528: /*
2529: * If we are clearing the modify bit and this page was marked EXEC
2530: * and the user of the page thinks the page was modified, then we
2531: * need to clean it from the icache if it's mapped or clear the EXEC
2532: * bit if it's not mapped. The page itself might not have the CHG
2533: * bit set if the modification was done via DMA to the page.
2534: */
2535: if ((ptebit & PTE_CHG) && (rv & PTE_EXEC)) {
2536: if (LIST_EMPTY(pvoh)) {
1.85 matt 2537: DPRINTFN(EXEC, "[pmap_clear_bit: %#" _PRIxpa ": clear-exec]\n",
2538: VM_PAGE_TO_PHYS(pg));
1.1 matt 2539: pmap_attr_clear(pg, PTE_EXEC);
2540: PMAPCOUNT(exec_uncached_clear_modify);
2541: } else {
1.85 matt 2542: DPRINTFN(EXEC, "[pmap_clear_bit: %#" _PRIxpa ": syncicache]\n",
2543: VM_PAGE_TO_PHYS(pg));
1.34 yamt 2544: pmap_syncicache(VM_PAGE_TO_PHYS(pg), PAGE_SIZE);
1.1 matt 2545: PMAPCOUNT(exec_synced_clear_modify);
2546: }
2547: }
1.50 ad 2548: PMAP_UNLOCK();
1.1 matt 2549: return (rv & ptebit) != 0;
2550: }
2551:
2552: void
2553: pmap_procwr(struct proc *p, vaddr_t va, size_t len)
2554: {
2555: struct pvo_entry *pvo;
2556: size_t offset = va & ADDR_POFF;
2557: int s;
2558:
1.50 ad 2559: PMAP_LOCK();
1.1 matt 2560: s = splvm();
2561: while (len > 0) {
1.6 thorpej 2562: size_t seglen = PAGE_SIZE - offset;
1.1 matt 2563: if (seglen > len)
2564: seglen = len;
2565: pvo = pmap_pvo_find_va(p->p_vmspace->vm_map.pmap, va, NULL);
1.39 matt 2566: if (pvo != NULL && PVO_EXECUTABLE_P(pvo)) {
1.1 matt 2567: pmap_syncicache(
2568: (pvo->pvo_pte.pte_lo & PTE_RPGN) | offset, seglen);
2569: PMAP_PVO_CHECK(pvo);
2570: }
2571: va += seglen;
2572: len -= seglen;
2573: offset = 0;
2574: }
2575: splx(s);
1.50 ad 2576: PMAP_UNLOCK();
1.1 matt 2577: }
2578:
2579: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
2580: void
1.2 matt 2581: pmap_pte_print(volatile struct pte *pt)
1.1 matt 2582: {
2583: printf("PTE %p: ", pt);
1.38 sanjayl 2584:
1.53 garbled 2585: #if defined(PMAP_OEA)
1.1 matt 2586: /* High word: */
1.54 mlelstv 2587: printf("%#" _PRIxpte ": [", pt->pte_hi);
1.53 garbled 2588: #else
1.54 mlelstv 2589: printf("%#" _PRIxpte ": [", pt->pte_hi);
1.53 garbled 2590: #endif /* PMAP_OEA */
1.38 sanjayl 2591:
1.1 matt 2592: printf("%c ", (pt->pte_hi & PTE_VALID) ? 'v' : 'i');
2593: printf("%c ", (pt->pte_hi & PTE_HID) ? 'h' : '-');
1.38 sanjayl 2594:
1.54 mlelstv 2595: printf("%#" _PRIxpte " %#" _PRIxpte "",
1.38 sanjayl 2596: (pt->pte_hi &~ PTE_VALID)>>PTE_VSID_SHFT,
2597: pt->pte_hi & PTE_API);
1.53 garbled 2598: #if defined(PMAP_OEA) || defined(PMAP_OEA64_BRIDGE)
1.54 mlelstv 2599: printf(" (va %#" _PRIxva ")] ", pmap_pte_to_va(pt));
1.38 sanjayl 2600: #else
1.54 mlelstv 2601: printf(" (va %#" _PRIxva ")] ", pmap_pte_to_va(pt));
1.53 garbled 2602: #endif /* PMAP_OEA */
1.38 sanjayl 2603:
1.1 matt 2604: /* Low word: */
1.53 garbled 2605: #if defined (PMAP_OEA)
1.54 mlelstv 2606: printf(" %#" _PRIxpte ": [", pt->pte_lo);
2607: printf("%#" _PRIxpte "... ", pt->pte_lo >> 12);
1.53 garbled 2608: #else
1.54 mlelstv 2609: printf(" %#" _PRIxpte ": [", pt->pte_lo);
2610: printf("%#" _PRIxpte "... ", pt->pte_lo >> 12);
1.38 sanjayl 2611: #endif
1.1 matt 2612: printf("%c ", (pt->pte_lo & PTE_REF) ? 'r' : 'u');
2613: printf("%c ", (pt->pte_lo & PTE_CHG) ? 'c' : 'n');
2614: printf("%c", (pt->pte_lo & PTE_W) ? 'w' : '.');
2615: printf("%c", (pt->pte_lo & PTE_I) ? 'i' : '.');
2616: printf("%c", (pt->pte_lo & PTE_M) ? 'm' : '.');
2617: printf("%c ", (pt->pte_lo & PTE_G) ? 'g' : '.');
2618: switch (pt->pte_lo & PTE_PP) {
2619: case PTE_BR: printf("br]\n"); break;
2620: case PTE_BW: printf("bw]\n"); break;
2621: case PTE_SO: printf("so]\n"); break;
2622: case PTE_SW: printf("sw]\n"); break;
2623: }
2624: }
2625: #endif
2626:
2627: #if defined(DDB)
2628: void
2629: pmap_pteg_check(void)
2630: {
1.2 matt 2631: volatile struct pte *pt;
1.1 matt 2632: int i;
2633: int ptegidx;
2634: u_int p_valid = 0;
2635: u_int s_valid = 0;
2636: u_int invalid = 0;
1.38 sanjayl 2637:
1.1 matt 2638: for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) {
2639: for (pt = pmap_pteg_table[ptegidx].pt, i = 8; --i >= 0; pt++) {
2640: if (pt->pte_hi & PTE_VALID) {
2641: if (pt->pte_hi & PTE_HID)
2642: s_valid++;
2643: else
1.38 sanjayl 2644: {
1.1 matt 2645: p_valid++;
1.38 sanjayl 2646: }
1.1 matt 2647: } else
2648: invalid++;
2649: }
2650: }
2651: printf("pteg_check: v(p) %#x (%d), v(s) %#x (%d), i %#x (%d)\n",
2652: p_valid, p_valid, s_valid, s_valid,
2653: invalid, invalid);
2654: }
2655:
2656: void
2657: pmap_print_mmuregs(void)
2658: {
2659: int i;
1.97 rin 2660: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1 matt 2661: u_int cpuvers;
1.90 mrg 2662: #endif
1.53 garbled 2663: #ifndef PMAP_OEA64
1.1 matt 2664: vaddr_t addr;
1.2 matt 2665: register_t soft_sr[16];
1.18 matt 2666: #endif
1.97 rin 2667: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1 matt 2668: struct bat soft_ibat[4];
2669: struct bat soft_dbat[4];
1.38 sanjayl 2670: #endif
1.53 garbled 2671: paddr_t sdr1;
1.1 matt 2672:
1.97 rin 2673: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1 matt 2674: cpuvers = MFPVR() >> 16;
1.90 mrg 2675: #endif
1.35 perry 2676: __asm volatile ("mfsdr1 %0" : "=r"(sdr1));
1.53 garbled 2677: #ifndef PMAP_OEA64
1.16 kleink 2678: addr = 0;
1.27 chs 2679: for (i = 0; i < 16; i++) {
1.1 matt 2680: soft_sr[i] = MFSRIN(addr);
2681: addr += (1 << ADDR_SR_SHFT);
2682: }
1.18 matt 2683: #endif
1.1 matt 2684:
1.97 rin 2685: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1 matt 2686: /* read iBAT (601: uBAT) registers */
1.35 perry 2687: __asm volatile ("mfibatu %0,0" : "=r"(soft_ibat[0].batu));
2688: __asm volatile ("mfibatl %0,0" : "=r"(soft_ibat[0].batl));
2689: __asm volatile ("mfibatu %0,1" : "=r"(soft_ibat[1].batu));
2690: __asm volatile ("mfibatl %0,1" : "=r"(soft_ibat[1].batl));
2691: __asm volatile ("mfibatu %0,2" : "=r"(soft_ibat[2].batu));
2692: __asm volatile ("mfibatl %0,2" : "=r"(soft_ibat[2].batl));
2693: __asm volatile ("mfibatu %0,3" : "=r"(soft_ibat[3].batu));
2694: __asm volatile ("mfibatl %0,3" : "=r"(soft_ibat[3].batl));
1.1 matt 2695:
2696:
2697: if (cpuvers != MPC601) {
2698: /* read dBAT registers */
1.35 perry 2699: __asm volatile ("mfdbatu %0,0" : "=r"(soft_dbat[0].batu));
2700: __asm volatile ("mfdbatl %0,0" : "=r"(soft_dbat[0].batl));
2701: __asm volatile ("mfdbatu %0,1" : "=r"(soft_dbat[1].batu));
2702: __asm volatile ("mfdbatl %0,1" : "=r"(soft_dbat[1].batl));
2703: __asm volatile ("mfdbatu %0,2" : "=r"(soft_dbat[2].batu));
2704: __asm volatile ("mfdbatl %0,2" : "=r"(soft_dbat[2].batl));
2705: __asm volatile ("mfdbatu %0,3" : "=r"(soft_dbat[3].batu));
2706: __asm volatile ("mfdbatl %0,3" : "=r"(soft_dbat[3].batl));
1.1 matt 2707: }
1.38 sanjayl 2708: #endif
1.1 matt 2709:
1.54 mlelstv 2710: printf("SDR1:\t%#" _PRIxpa "\n", sdr1);
1.53 garbled 2711: #ifndef PMAP_OEA64
1.1 matt 2712: printf("SR[]:\t");
1.27 chs 2713: for (i = 0; i < 4; i++)
1.53 garbled 2714: printf("0x%08lx, ", soft_sr[i]);
1.1 matt 2715: printf("\n\t");
1.27 chs 2716: for ( ; i < 8; i++)
1.53 garbled 2717: printf("0x%08lx, ", soft_sr[i]);
1.1 matt 2718: printf("\n\t");
1.27 chs 2719: for ( ; i < 12; i++)
1.53 garbled 2720: printf("0x%08lx, ", soft_sr[i]);
1.1 matt 2721: printf("\n\t");
1.27 chs 2722: for ( ; i < 16; i++)
1.53 garbled 2723: printf("0x%08lx, ", soft_sr[i]);
1.1 matt 2724: printf("\n");
1.18 matt 2725: #endif
1.1 matt 2726:
1.97 rin 2727: #if defined(PMAP_OEA) || defined(PMAP_OEA64_BRIDGE)
1.1 matt 2728: printf("%cBAT[]:\t", cpuvers == MPC601 ? 'u' : 'i');
1.27 chs 2729: for (i = 0; i < 4; i++) {
1.2 matt 2730: printf("0x%08lx 0x%08lx, ",
1.1 matt 2731: soft_ibat[i].batu, soft_ibat[i].batl);
2732: if (i == 1)
2733: printf("\n\t");
2734: }
2735: if (cpuvers != MPC601) {
2736: printf("\ndBAT[]:\t");
1.27 chs 2737: for (i = 0; i < 4; i++) {
1.2 matt 2738: printf("0x%08lx 0x%08lx, ",
1.1 matt 2739: soft_dbat[i].batu, soft_dbat[i].batl);
2740: if (i == 1)
2741: printf("\n\t");
2742: }
2743: }
2744: printf("\n");
1.53 garbled 2745: #endif /* PMAP_OEA... */
1.1 matt 2746: }
2747:
2748: void
2749: pmap_print_pte(pmap_t pm, vaddr_t va)
2750: {
2751: struct pvo_entry *pvo;
1.2 matt 2752: volatile struct pte *pt;
1.1 matt 2753: int pteidx;
2754:
2755: pvo = pmap_pvo_find_va(pm, va, &pteidx);
2756: if (pvo != NULL) {
2757: pt = pmap_pvo_to_pte(pvo, pteidx);
2758: if (pt != NULL) {
1.53 garbled 2759: printf("VA %#" _PRIxva " -> %p -> %s %#" _PRIxpte ", %#" _PRIxpte "\n",
1.38 sanjayl 2760: va, pt,
2761: pt->pte_hi & PTE_HID ? "(sec)" : "(pri)",
2762: pt->pte_hi, pt->pte_lo);
1.1 matt 2763: } else {
2764: printf("No valid PTE found\n");
2765: }
2766: } else {
2767: printf("Address not in pmap\n");
2768: }
2769: }
2770:
2771: void
2772: pmap_pteg_dist(void)
2773: {
2774: struct pvo_entry *pvo;
2775: int ptegidx;
2776: int depth;
2777: int max_depth = 0;
2778: unsigned int depths[64];
2779:
2780: memset(depths, 0, sizeof(depths));
2781: for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) {
2782: depth = 0;
2783: TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
2784: depth++;
2785: }
2786: if (depth > max_depth)
2787: max_depth = depth;
2788: if (depth > 63)
2789: depth = 63;
2790: depths[depth]++;
2791: }
2792:
2793: for (depth = 0; depth < 64; depth++) {
2794: printf(" [%2d]: %8u", depth, depths[depth]);
2795: if ((depth & 3) == 3)
2796: printf("\n");
2797: if (depth == max_depth)
2798: break;
2799: }
2800: if ((depth & 3) != 3)
2801: printf("\n");
2802: printf("Max depth found was %d\n", max_depth);
2803: }
2804: #endif /* DEBUG */
2805:
2806: #if defined(PMAPCHECK) || defined(DEBUG)
2807: void
2808: pmap_pvo_verify(void)
2809: {
2810: int ptegidx;
2811: int s;
2812:
2813: s = splvm();
2814: for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) {
2815: struct pvo_entry *pvo;
2816: TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
2817: if ((uintptr_t) pvo >= SEGMENT_LENGTH)
2818: panic("pmap_pvo_verify: invalid pvo %p "
2819: "on list %#x", pvo, ptegidx);
2820: pmap_pvo_check(pvo);
2821: }
2822: }
2823: splx(s);
2824: }
2825: #endif /* PMAPCHECK */
2826:
2827:
2828: void *
2829: pmap_pool_ualloc(struct pool *pp, int flags)
2830: {
2831: struct pvo_page *pvop;
2832:
1.50 ad 2833: if (uvm.page_init_done != true) {
2834: return (void *) uvm_pageboot_alloc(PAGE_SIZE);
2835: }
2836:
2837: PMAP_LOCK();
1.1 matt 2838: pvop = SIMPLEQ_FIRST(&pmap_upvop_head);
2839: if (pvop != NULL) {
2840: pmap_upvop_free--;
2841: SIMPLEQ_REMOVE_HEAD(&pmap_upvop_head, pvop_link);
1.50 ad 2842: PMAP_UNLOCK();
1.1 matt 2843: return pvop;
2844: }
1.50 ad 2845: PMAP_UNLOCK();
1.1 matt 2846: return pmap_pool_malloc(pp, flags);
2847: }
2848:
2849: void *
2850: pmap_pool_malloc(struct pool *pp, int flags)
2851: {
2852: struct pvo_page *pvop;
2853: struct vm_page *pg;
2854:
1.50 ad 2855: PMAP_LOCK();
1.1 matt 2856: pvop = SIMPLEQ_FIRST(&pmap_mpvop_head);
2857: if (pvop != NULL) {
2858: pmap_mpvop_free--;
2859: SIMPLEQ_REMOVE_HEAD(&pmap_mpvop_head, pvop_link);
1.50 ad 2860: PMAP_UNLOCK();
1.1 matt 2861: return pvop;
2862: }
1.50 ad 2863: PMAP_UNLOCK();
1.1 matt 2864: again:
2865: pg = uvm_pagealloc_strat(NULL, 0, NULL, UVM_PGA_USERESERVE,
2866: UVM_PGA_STRAT_ONLY, VM_FREELIST_FIRST256);
2867: if (__predict_false(pg == NULL)) {
2868: if (flags & PR_WAITOK) {
2869: uvm_wait("plpg");
2870: goto again;
2871: } else {
2872: return (0);
2873: }
2874: }
1.53 garbled 2875: KDASSERT(VM_PAGE_TO_PHYS(pg) == (uintptr_t)VM_PAGE_TO_PHYS(pg));
2876: return (void *)(uintptr_t) VM_PAGE_TO_PHYS(pg);
1.1 matt 2877: }
2878:
2879: void
2880: pmap_pool_ufree(struct pool *pp, void *va)
2881: {
2882: struct pvo_page *pvop;
2883: #if 0
2884: if (PHYS_TO_VM_PAGE((paddr_t) va) != NULL) {
2885: pmap_pool_mfree(va, size, tag);
2886: return;
2887: }
2888: #endif
1.50 ad 2889: PMAP_LOCK();
1.1 matt 2890: pvop = va;
2891: SIMPLEQ_INSERT_HEAD(&pmap_upvop_head, pvop, pvop_link);
2892: pmap_upvop_free++;
2893: if (pmap_upvop_free > pmap_upvop_maxfree)
2894: pmap_upvop_maxfree = pmap_upvop_free;
1.50 ad 2895: PMAP_UNLOCK();
1.1 matt 2896: }
2897:
2898: void
2899: pmap_pool_mfree(struct pool *pp, void *va)
2900: {
2901: struct pvo_page *pvop;
2902:
1.50 ad 2903: PMAP_LOCK();
1.1 matt 2904: pvop = va;
2905: SIMPLEQ_INSERT_HEAD(&pmap_mpvop_head, pvop, pvop_link);
2906: pmap_mpvop_free++;
2907: if (pmap_mpvop_free > pmap_mpvop_maxfree)
2908: pmap_mpvop_maxfree = pmap_mpvop_free;
1.50 ad 2909: PMAP_UNLOCK();
1.1 matt 2910: #if 0
2911: uvm_pagefree(PHYS_TO_VM_PAGE((paddr_t) va));
2912: #endif
2913: }
2914:
2915: /*
2916: * This routine in bootstraping to steal to-be-managed memory (which will
2917: * then be unmanaged). We use it to grab from the first 256MB for our
2918: * pmap needs and above 256MB for other stuff.
2919: */
2920: vaddr_t
1.10 thorpej 2921: pmap_steal_memory(vsize_t vsize, vaddr_t *vstartp, vaddr_t *vendp)
1.1 matt 2922: {
2923: vsize_t size;
2924: vaddr_t va;
1.94 cherry 2925: paddr_t start, end, pa = 0;
2926: int npgs, freelist;
2927: uvm_physseg_t bank;
1.1 matt 2928:
1.45 thorpej 2929: if (uvm.page_init_done == true)
1.1 matt 2930: panic("pmap_steal_memory: called _after_ bootstrap");
2931:
1.10 thorpej 2932: *vstartp = VM_MIN_KERNEL_ADDRESS;
2933: *vendp = VM_MAX_KERNEL_ADDRESS;
2934:
1.1 matt 2935: size = round_page(vsize);
2936: npgs = atop(size);
2937:
2938: /*
2939: * PA 0 will never be among those given to UVM so we can use it
2940: * to indicate we couldn't steal any memory.
2941: */
1.94 cherry 2942:
2943: for (bank = uvm_physseg_get_first();
2944: uvm_physseg_valid_p(bank);
2945: bank = uvm_physseg_get_next(bank)) {
2946:
2947: freelist = uvm_physseg_get_free_list(bank);
2948: start = uvm_physseg_get_start(bank);
2949: end = uvm_physseg_get_end(bank);
2950:
2951: if (freelist == VM_FREELIST_FIRST256 &&
2952: (end - start) >= npgs) {
2953: pa = ptoa(start);
1.1 matt 2954: break;
2955: }
2956: }
2957:
2958: if (pa == 0)
2959: panic("pmap_steal_memory: no approriate memory to steal!");
2960:
1.94 cherry 2961: uvm_physseg_unplug(start, npgs);
1.1 matt 2962:
2963: va = (vaddr_t) pa;
1.46 christos 2964: memset((void *) va, 0, size);
1.1 matt 2965: pmap_pages_stolen += npgs;
2966: #ifdef DEBUG
2967: if (pmapdebug && npgs > 1) {
2968: u_int cnt = 0;
1.94 cherry 2969: for (bank = uvm_physseg_get_first();
2970: uvm_physseg_valid_p(bank);
2971: bank = uvm_physseg_get_next(bank)) {
2972: cnt += uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank);
1.73 uebayasi 2973: }
1.1 matt 2974: printf("pmap_steal_memory: stole %u (total %u) pages (%u left)\n",
2975: npgs, pmap_pages_stolen, cnt);
2976: }
2977: #endif
2978:
2979: return va;
2980: }
2981:
2982: /*
2983: * Find a chuck of memory with right size and alignment.
2984: */
1.53 garbled 2985: paddr_t
1.1 matt 2986: pmap_boot_find_memory(psize_t size, psize_t alignment, int at_end)
2987: {
2988: struct mem_region *mp;
2989: paddr_t s, e;
2990: int i, j;
2991:
2992: size = round_page(size);
2993:
2994: DPRINTFN(BOOT,
1.85 matt 2995: "pmap_boot_find_memory: size=%#" _PRIxpa ", alignment=%#" _PRIxpa ", at_end=%d",
2996: size, alignment, at_end);
1.1 matt 2997:
1.6 thorpej 2998: if (alignment < PAGE_SIZE || (alignment & (alignment-1)) != 0)
1.54 mlelstv 2999: panic("pmap_boot_find_memory: invalid alignment %#" _PRIxpa,
1.1 matt 3000: alignment);
3001:
3002: if (at_end) {
1.6 thorpej 3003: if (alignment != PAGE_SIZE)
1.1 matt 3004: panic("pmap_boot_find_memory: invalid ending "
1.53 garbled 3005: "alignment %#" _PRIxpa, alignment);
1.1 matt 3006:
3007: for (mp = &avail[avail_cnt-1]; mp >= avail; mp--) {
3008: s = mp->start + mp->size - size;
3009: if (s >= mp->start && mp->size >= size) {
1.85 matt 3010: DPRINTFN(BOOT, ": %#" _PRIxpa "\n", s);
1.1 matt 3011: DPRINTFN(BOOT,
1.85 matt 3012: "pmap_boot_find_memory: b-avail[%d] start "
3013: "%#" _PRIxpa " size %#" _PRIxpa "\n", mp - avail,
3014: mp->start, mp->size);
1.1 matt 3015: mp->size -= size;
3016: DPRINTFN(BOOT,
1.85 matt 3017: "pmap_boot_find_memory: a-avail[%d] start "
3018: "%#" _PRIxpa " size %#" _PRIxpa "\n", mp - avail,
3019: mp->start, mp->size);
1.53 garbled 3020: return s;
1.1 matt 3021: }
3022: }
3023: panic("pmap_boot_find_memory: no available memory");
3024: }
3025:
3026: for (mp = avail, i = 0; i < avail_cnt; i++, mp++) {
3027: s = (mp->start + alignment - 1) & ~(alignment-1);
3028: e = s + size;
3029:
3030: /*
3031: * Is the calculated region entirely within the region?
3032: */
3033: if (s < mp->start || e > mp->start + mp->size)
3034: continue;
3035:
1.85 matt 3036: DPRINTFN(BOOT, ": %#" _PRIxpa "\n", s);
1.1 matt 3037: if (s == mp->start) {
3038: /*
3039: * If the block starts at the beginning of region,
3040: * adjust the size & start. (the region may now be
3041: * zero in length)
3042: */
3043: DPRINTFN(BOOT,
1.85 matt 3044: "pmap_boot_find_memory: b-avail[%d] start "
3045: "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1 matt 3046: mp->start += size;
3047: mp->size -= size;
3048: DPRINTFN(BOOT,
1.85 matt 3049: "pmap_boot_find_memory: a-avail[%d] start "
3050: "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1 matt 3051: } else if (e == mp->start + mp->size) {
3052: /*
3053: * If the block starts at the beginning of region,
3054: * adjust only the size.
3055: */
3056: DPRINTFN(BOOT,
1.85 matt 3057: "pmap_boot_find_memory: b-avail[%d] start "
3058: "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1 matt 3059: mp->size -= size;
3060: DPRINTFN(BOOT,
1.85 matt 3061: "pmap_boot_find_memory: a-avail[%d] start "
3062: "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1 matt 3063: } else {
3064: /*
3065: * Block is in the middle of the region, so we
3066: * have to split it in two.
3067: */
3068: for (j = avail_cnt; j > i + 1; j--) {
3069: avail[j] = avail[j-1];
3070: }
3071: DPRINTFN(BOOT,
1.85 matt 3072: "pmap_boot_find_memory: b-avail[%d] start "
3073: "%#" _PRIxpa " size %#" _PRIxpa "\n", i, mp->start, mp->size);
1.1 matt 3074: mp[1].start = e;
3075: mp[1].size = mp[0].start + mp[0].size - e;
3076: mp[0].size = s - mp[0].start;
3077: avail_cnt++;
3078: for (; i < avail_cnt; i++) {
3079: DPRINTFN(BOOT,
1.85 matt 3080: "pmap_boot_find_memory: a-avail[%d] "
3081: "start %#" _PRIxpa " size %#" _PRIxpa "\n", i,
3082: avail[i].start, avail[i].size);
1.1 matt 3083: }
3084: }
1.53 garbled 3085: KASSERT(s == (uintptr_t) s);
3086: return s;
1.1 matt 3087: }
3088: panic("pmap_boot_find_memory: not enough memory for "
1.54 mlelstv 3089: "%#" _PRIxpa "/%#" _PRIxpa " allocation?", size, alignment);
1.1 matt 3090: }
3091:
1.38 sanjayl 3092: /* XXXSL: we dont have any BATs to do this, map in Segment 0 1:1 using page tables */
1.53 garbled 3093: #if defined (PMAP_OEA64_BRIDGE)
1.38 sanjayl 3094: int
3095: pmap_setup_segment0_map(int use_large_pages, ...)
3096: {
1.88 christos 3097: vaddr_t va, va_end;
1.38 sanjayl 3098:
3099: register_t pte_lo = 0x0;
1.90 mrg 3100: int ptegidx = 0;
1.38 sanjayl 3101: struct pte pte;
3102: va_list ap;
3103:
3104: /* Coherent + Supervisor RW, no user access */
3105: pte_lo = PTE_M;
3106:
3107: /* XXXSL
3108: * Map in 1st segment 1:1, we'll be careful not to spill kernel entries later,
3109: * these have to take priority.
3110: */
3111: for (va = 0x0; va < SEGMENT_LENGTH; va += 0x1000) {
3112: ptegidx = va_to_pteg(pmap_kernel(), va);
3113: pmap_pte_create(&pte, pmap_kernel(), va, va | pte_lo);
1.90 mrg 3114: (void)pmap_pte_insert(ptegidx, &pte);
1.38 sanjayl 3115: }
3116:
3117: va_start(ap, use_large_pages);
3118: while (1) {
3119: paddr_t pa;
3120: size_t size;
3121:
3122: va = va_arg(ap, vaddr_t);
3123:
3124: if (va == 0)
3125: break;
3126:
3127: pa = va_arg(ap, paddr_t);
3128: size = va_arg(ap, size_t);
3129:
1.88 christos 3130: for (va_end = va + size; va < va_end; va += 0x1000, pa += 0x1000) {
1.38 sanjayl 3131: #if 0
1.54 mlelstv 3132: printf("%s: Inserting: va: %#" _PRIxva ", pa: %#" _PRIxpa "\n", __func__, va, pa);
1.38 sanjayl 3133: #endif
3134: ptegidx = va_to_pteg(pmap_kernel(), va);
3135: pmap_pte_create(&pte, pmap_kernel(), va, pa | pte_lo);
1.90 mrg 3136: (void)pmap_pte_insert(ptegidx, &pte);
1.38 sanjayl 3137: }
3138: }
1.93 dholland 3139: va_end(ap);
1.38 sanjayl 3140:
3141: TLBSYNC();
3142: SYNC();
3143: return (0);
3144: }
1.53 garbled 3145: #endif /* PMAP_OEA64_BRIDGE */
1.38 sanjayl 3146:
1.1 matt 3147: /*
1.99 ! thorpej 3148: * Set up the bottom level of the data structures necessary for the kernel
! 3149: * to manage memory. MMU hardware is programmed in pmap_bootstrap2().
1.1 matt 3150: */
3151: void
1.99 ! thorpej 3152: pmap_bootstrap1(paddr_t kernelstart, paddr_t kernelend)
1.1 matt 3153: {
3154: struct mem_region *mp, tmp;
3155: paddr_t s, e;
3156: psize_t size;
3157: int i, j;
3158:
3159: /*
3160: * Get memory.
3161: */
3162: mem_regions(&mem, &avail);
3163: #if defined(DEBUG)
3164: if (pmapdebug & PMAPDEBUG_BOOT) {
3165: printf("pmap_bootstrap: memory configuration:\n");
3166: for (mp = mem; mp->size; mp++) {
1.54 mlelstv 3167: printf("pmap_bootstrap: mem start %#" _PRIxpa " size %#" _PRIxpa "\n",
1.1 matt 3168: mp->start, mp->size);
3169: }
3170: for (mp = avail; mp->size; mp++) {
1.54 mlelstv 3171: printf("pmap_bootstrap: avail start %#" _PRIxpa " size %#" _PRIxpa "\n",
1.1 matt 3172: mp->start, mp->size);
3173: }
3174: }
3175: #endif
3176:
3177: /*
3178: * Find out how much physical memory we have and in how many chunks.
3179: */
3180: for (mem_cnt = 0, mp = mem; mp->size; mp++) {
3181: if (mp->start >= pmap_memlimit)
3182: continue;
3183: if (mp->start + mp->size > pmap_memlimit) {
3184: size = pmap_memlimit - mp->start;
3185: physmem += btoc(size);
3186: } else {
3187: physmem += btoc(mp->size);
3188: }
3189: mem_cnt++;
3190: }
3191:
3192: /*
3193: * Count the number of available entries.
3194: */
3195: for (avail_cnt = 0, mp = avail; mp->size; mp++)
3196: avail_cnt++;
3197:
3198: /*
3199: * Page align all regions.
3200: */
3201: kernelstart = trunc_page(kernelstart);
3202: kernelend = round_page(kernelend);
3203: for (mp = avail, i = 0; i < avail_cnt; i++, mp++) {
3204: s = round_page(mp->start);
3205: mp->size -= (s - mp->start);
3206: mp->size = trunc_page(mp->size);
3207: mp->start = s;
3208: e = mp->start + mp->size;
3209:
3210: DPRINTFN(BOOT,
1.85 matt 3211: "pmap_bootstrap: b-avail[%d] start %#" _PRIxpa " size %#" _PRIxpa "\n",
3212: i, mp->start, mp->size);
1.1 matt 3213:
3214: /*
3215: * Don't allow the end to run beyond our artificial limit
3216: */
3217: if (e > pmap_memlimit)
3218: e = pmap_memlimit;
3219:
3220: /*
3221: * Is this region empty or strange? skip it.
3222: */
3223: if (e <= s) {
3224: mp->start = 0;
3225: mp->size = 0;
3226: continue;
3227: }
3228:
3229: /*
3230: * Does this overlap the beginning of kernel?
3231: * Does extend past the end of the kernel?
3232: */
3233: else if (s < kernelstart && e > kernelstart) {
3234: if (e > kernelend) {
3235: avail[avail_cnt].start = kernelend;
3236: avail[avail_cnt].size = e - kernelend;
3237: avail_cnt++;
3238: }
3239: mp->size = kernelstart - s;
3240: }
3241: /*
3242: * Check whether this region overlaps the end of the kernel.
3243: */
3244: else if (s < kernelend && e > kernelend) {
3245: mp->start = kernelend;
3246: mp->size = e - kernelend;
3247: }
3248: /*
3249: * Look whether this regions is completely inside the kernel.
3250: * Nuke it if it does.
3251: */
3252: else if (s >= kernelstart && e <= kernelend) {
3253: mp->start = 0;
3254: mp->size = 0;
3255: }
3256: /*
3257: * If the user imposed a memory limit, enforce it.
3258: */
3259: else if (s >= pmap_memlimit) {
1.6 thorpej 3260: mp->start = -PAGE_SIZE; /* let's know why */
1.1 matt 3261: mp->size = 0;
3262: }
3263: else {
3264: mp->start = s;
3265: mp->size = e - s;
3266: }
3267: DPRINTFN(BOOT,
1.85 matt 3268: "pmap_bootstrap: a-avail[%d] start %#" _PRIxpa " size %#" _PRIxpa "\n",
3269: i, mp->start, mp->size);
1.1 matt 3270: }
3271:
3272: /*
3273: * Move (and uncount) all the null return to the end.
3274: */
3275: for (mp = avail, i = 0; i < avail_cnt; i++, mp++) {
3276: if (mp->size == 0) {
3277: tmp = avail[i];
3278: avail[i] = avail[--avail_cnt];
3279: avail[avail_cnt] = avail[i];
3280: }
3281: }
3282:
3283: /*
1.61 skrll 3284: * (Bubble)sort them into ascending order.
1.1 matt 3285: */
3286: for (i = 0; i < avail_cnt; i++) {
3287: for (j = i + 1; j < avail_cnt; j++) {
3288: if (avail[i].start > avail[j].start) {
3289: tmp = avail[i];
3290: avail[i] = avail[j];
3291: avail[j] = tmp;
3292: }
3293: }
3294: }
3295:
3296: /*
3297: * Make sure they don't overlap.
3298: */
3299: for (mp = avail, i = 0; i < avail_cnt - 1; i++, mp++) {
3300: if (mp[0].start + mp[0].size > mp[1].start) {
3301: mp[0].size = mp[1].start - mp[0].start;
3302: }
3303: DPRINTFN(BOOT,
1.85 matt 3304: "pmap_bootstrap: avail[%d] start %#" _PRIxpa " size %#" _PRIxpa "\n",
3305: i, mp->start, mp->size);
1.1 matt 3306: }
3307: DPRINTFN(BOOT,
1.85 matt 3308: "pmap_bootstrap: avail[%d] start %#" _PRIxpa " size %#" _PRIxpa "\n",
3309: i, mp->start, mp->size);
1.1 matt 3310:
3311: #ifdef PTEGCOUNT
3312: pmap_pteg_cnt = PTEGCOUNT;
3313: #else /* PTEGCOUNT */
1.38 sanjayl 3314:
1.1 matt 3315: pmap_pteg_cnt = 0x1000;
3316:
3317: while (pmap_pteg_cnt < physmem)
3318: pmap_pteg_cnt <<= 1;
3319:
3320: pmap_pteg_cnt >>= 1;
3321: #endif /* PTEGCOUNT */
3322:
1.38 sanjayl 3323: #ifdef DEBUG
1.85 matt 3324: DPRINTFN(BOOT, "pmap_pteg_cnt: 0x%x\n", pmap_pteg_cnt);
1.38 sanjayl 3325: #endif
3326:
1.1 matt 3327: /*
3328: * Find suitably aligned memory for PTEG hash table.
3329: */
1.2 matt 3330: size = pmap_pteg_cnt * sizeof(struct pteg);
1.53 garbled 3331: pmap_pteg_table = (void *)(uintptr_t) pmap_boot_find_memory(size, size, 0);
1.38 sanjayl 3332:
3333: #ifdef DEBUG
3334: DPRINTFN(BOOT,
1.85 matt 3335: "PTEG cnt: 0x%x HTAB size: 0x%08x bytes, address: %p\n", pmap_pteg_cnt, (unsigned int)size, pmap_pteg_table);
1.38 sanjayl 3336: #endif
3337:
3338:
1.1 matt 3339: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
3340: if ( (uintptr_t) pmap_pteg_table + size > SEGMENT_LENGTH)
1.54 mlelstv 3341: panic("pmap_bootstrap: pmap_pteg_table end (%p + %#" _PRIxpa ") > 256MB",
1.1 matt 3342: pmap_pteg_table, size);
3343: #endif
3344:
1.32 he 3345: memset(__UNVOLATILE(pmap_pteg_table), 0,
3346: pmap_pteg_cnt * sizeof(struct pteg));
1.1 matt 3347: pmap_pteg_mask = pmap_pteg_cnt - 1;
3348:
3349: /*
3350: * We cannot do pmap_steal_memory here since UVM hasn't been loaded
3351: * with pages. So we just steal them before giving them to UVM.
3352: */
3353: size = sizeof(pmap_pvo_table[0]) * pmap_pteg_cnt;
1.53 garbled 3354: pmap_pvo_table = (void *)(uintptr_t) pmap_boot_find_memory(size, PAGE_SIZE, 0);
1.1 matt 3355: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
3356: if ( (uintptr_t) pmap_pvo_table + size > SEGMENT_LENGTH)
1.54 mlelstv 3357: panic("pmap_bootstrap: pmap_pvo_table end (%p + %#" _PRIxpa ") > 256MB",
1.1 matt 3358: pmap_pvo_table, size);
3359: #endif
3360:
3361: for (i = 0; i < pmap_pteg_cnt; i++)
3362: TAILQ_INIT(&pmap_pvo_table[i]);
3363:
3364: #ifndef MSGBUFADDR
3365: /*
3366: * Allocate msgbuf in high memory.
3367: */
1.53 garbled 3368: msgbuf_paddr = pmap_boot_find_memory(MSGBUFSIZE, PAGE_SIZE, 1);
1.1 matt 3369: #endif
3370:
3371: for (mp = avail, i = 0; i < avail_cnt; mp++, i++) {
3372: paddr_t pfstart = atop(mp->start);
3373: paddr_t pfend = atop(mp->start + mp->size);
3374: if (mp->size == 0)
3375: continue;
3376: if (mp->start + mp->size <= SEGMENT_LENGTH) {
3377: uvm_page_physload(pfstart, pfend, pfstart, pfend,
3378: VM_FREELIST_FIRST256);
3379: } else if (mp->start >= SEGMENT_LENGTH) {
3380: uvm_page_physload(pfstart, pfend, pfstart, pfend,
3381: VM_FREELIST_DEFAULT);
3382: } else {
3383: pfend = atop(SEGMENT_LENGTH);
3384: uvm_page_physload(pfstart, pfend, pfstart, pfend,
3385: VM_FREELIST_FIRST256);
3386: pfstart = atop(SEGMENT_LENGTH);
3387: pfend = atop(mp->start + mp->size);
3388: uvm_page_physload(pfstart, pfend, pfstart, pfend,
3389: VM_FREELIST_DEFAULT);
3390: }
3391: }
3392:
3393: /*
3394: * Make sure kernel vsid is allocated as well as VSID 0.
3395: */
3396: pmap_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS-1)) / VSID_NBPW]
3397: |= 1 << (KERNEL_VSIDBITS % VSID_NBPW);
1.53 garbled 3398: pmap_vsid_bitmap[(PHYSMAP_VSIDBITS & (NPMAPS-1)) / VSID_NBPW]
3399: |= 1 << (PHYSMAP_VSIDBITS % VSID_NBPW);
1.1 matt 3400: pmap_vsid_bitmap[0] |= 1;
3401:
3402: /*
3403: * Initialize kernel pmap and hardware.
3404: */
1.38 sanjayl 3405:
1.53 garbled 3406: /* PMAP_OEA64_BRIDGE does support these instructions */
3407: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
1.1 matt 3408: for (i = 0; i < 16; i++) {
1.91 macallan 3409: #if defined(PPC_OEA601)
3410: /* XXX wedges for segment register 0xf , so set later */
3411: if ((iosrtable[i] & SR601_T) && ((MFPVR() >> 16) == MPC601))
3412: continue;
3413: #endif
1.38 sanjayl 3414: pmap_kernel()->pm_sr[i] = KERNELN_SEGMENT(i)|SR_PRKEY;
1.1 matt 3415: }
3416:
3417: pmap_kernel()->pm_sr[KERNEL_SR] = KERNEL_SEGMENT|SR_SUKEY|SR_PRKEY;
3418: #ifdef KERNEL2_SR
3419: pmap_kernel()->pm_sr[KERNEL2_SR] = KERNEL2_SEGMENT|SR_SUKEY|SR_PRKEY;
3420: #endif
1.53 garbled 3421: #endif /* PMAP_OEA || PMAP_OEA64_BRIDGE */
3422: #if defined (PMAP_OEA)
1.1 matt 3423: for (i = 0; i < 16; i++) {
3424: if (iosrtable[i] & SR601_T) {
3425: pmap_kernel()->pm_sr[i] = iosrtable[i];
3426: }
3427: }
1.38 sanjayl 3428: #endif
1.1 matt 3429:
3430: #ifdef ALTIVEC
3431: pmap_use_altivec = cpu_altivec;
3432: #endif
3433:
3434: #ifdef DEBUG
3435: if (pmapdebug & PMAPDEBUG_BOOT) {
3436: u_int cnt;
1.94 cherry 3437: uvm_physseg_t bank;
1.1 matt 3438: char pbuf[9];
1.94 cherry 3439: for (cnt = 0, bank = uvm_physseg_get_first();
3440: uvm_physseg_valid_p(bank);
3441: bank = uvm_physseg_get_next(bank)) {
3442: cnt += uvm_physseg_get_avail_end(bank) -
3443: uvm_physseg_get_avail_start(bank);
1.53 garbled 3444: printf("pmap_bootstrap: vm_physmem[%d]=%#" _PRIxpa "-%#" _PRIxpa "/%#" _PRIxpa "\n",
1.1 matt 3445: bank,
1.94 cherry 3446: ptoa(uvm_physseg_get_avail_start(bank)),
3447: ptoa(uvm_physseg_get_avail_end(bank)),
3448: ptoa(uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank)));
1.1 matt 3449: }
3450: format_bytes(pbuf, sizeof(pbuf), ptoa((u_int64_t) cnt));
3451: printf("pmap_bootstrap: UVM memory = %s (%u pages)\n",
3452: pbuf, cnt);
3453: }
3454: #endif
3455:
3456: pool_init(&pmap_upvo_pool, sizeof(struct pvo_entry),
3457: sizeof(struct pvo_entry), 0, 0, "pmap_upvopl",
1.60 chs 3458: &pmap_pool_uallocator, IPL_VM);
1.1 matt 3459:
3460: pool_setlowat(&pmap_upvo_pool, 252);
3461:
3462: pool_init(&pmap_pool, sizeof(struct pmap),
1.48 ad 3463: sizeof(void *), 0, 0, "pmap_pl", &pmap_pool_uallocator,
3464: IPL_NONE);
1.41 matt 3465:
1.89 macallan 3466: #if defined(PMAP_NEED_MAPKERNEL)
1.41 matt 3467: {
1.53 garbled 3468: struct pmap *pm = pmap_kernel();
1.58 garbled 3469: #if defined(PMAP_NEED_FULL_MAPKERNEL)
1.41 matt 3470: extern int etext[], kernel_text[];
3471: vaddr_t va, va_etext = (paddr_t) etext;
1.53 garbled 3472: #endif
3473: paddr_t pa, pa_end;
1.42 matt 3474: register_t sr;
1.53 garbled 3475: struct pte pt;
3476: unsigned int ptegidx;
3477: int bank;
1.42 matt 3478:
1.53 garbled 3479: sr = PHYSMAPN_SEGMENT(0) | SR_SUKEY|SR_PRKEY;
3480: pm->pm_sr[0] = sr;
1.41 matt 3481:
1.53 garbled 3482: for (bank = 0; bank < vm_nphysseg; bank++) {
1.73 uebayasi 3483: pa_end = ptoa(VM_PHYSMEM_PTR(bank)->avail_end);
3484: pa = ptoa(VM_PHYSMEM_PTR(bank)->avail_start);
1.53 garbled 3485: for (; pa < pa_end; pa += PAGE_SIZE) {
3486: ptegidx = va_to_pteg(pm, pa);
3487: pmap_pte_create(&pt, pm, pa, pa | PTE_M|PTE_BW);
3488: pmap_pte_insert(ptegidx, &pt);
3489: }
3490: }
3491:
1.58 garbled 3492: #if defined(PMAP_NEED_FULL_MAPKERNEL)
1.41 matt 3493: va = (vaddr_t) kernel_text;
3494:
3495: for (pa = kernelstart; va < va_etext;
1.53 garbled 3496: pa += PAGE_SIZE, va += PAGE_SIZE) {
3497: ptegidx = va_to_pteg(pm, va);
3498: pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BR);
3499: pmap_pte_insert(ptegidx, &pt);
3500: }
1.41 matt 3501:
3502: for (; pa < kernelend;
1.53 garbled 3503: pa += PAGE_SIZE, va += PAGE_SIZE) {
3504: ptegidx = va_to_pteg(pm, va);
3505: pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BW);
3506: pmap_pte_insert(ptegidx, &pt);
3507: }
3508:
1.58 garbled 3509: for (va = 0, pa = 0; va < kernelstart;
1.53 garbled 3510: pa += PAGE_SIZE, va += PAGE_SIZE) {
3511: ptegidx = va_to_pteg(pm, va);
1.58 garbled 3512: if (va < 0x3000)
3513: pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BR);
3514: else
3515: pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BW);
3516: pmap_pte_insert(ptegidx, &pt);
3517: }
3518: for (va = kernelend, pa = kernelend; va < SEGMENT_LENGTH;
3519: pa += PAGE_SIZE, va += PAGE_SIZE) {
3520: ptegidx = va_to_pteg(pm, va);
1.53 garbled 3521: pmap_pte_create(&pt, pm, va, pa | PTE_M|PTE_BW);
3522: pmap_pte_insert(ptegidx, &pt);
3523: }
3524: #endif
1.99 ! thorpej 3525: }
! 3526: #endif
! 3527: }
1.42 matt 3528:
1.99 ! thorpej 3529: /*
! 3530: * Using the data structures prepared in pmap_bootstrap1(), program
! 3531: * the MMU hardware.
! 3532: */
! 3533: void
! 3534: pmap_bootstrap2(void)
! 3535: {
! 3536: /* PMAP_OEA64_BRIDGE does support these instructions */
! 3537: #if defined (PMAP_OEA) || defined (PMAP_OEA64_BRIDGE)
! 3538: for (int i = 0; i < 16; i++) {
! 3539: #if defined(PPC_OEA601)
! 3540: /* XXX wedges for segment register 0xf , so set later */
! 3541: if ((iosrtable[i] & SR601_T) && ((MFPVR() >> 16) == MPC601))
! 3542: continue;
! 3543: #endif /* PPC_OEA601 */
! 3544: __asm volatile("mtsrin %0,%1"
! 3545: :: "r"(pmap_kernel()->pm_sr[i]),
! 3546: "r"(i << ADDR_SR_SHFT));
1.41 matt 3547: }
1.99 ! thorpej 3548: #endif /* PMAP_OEA || PMAP_OEA64_BRIDGE */
! 3549: #if defined (PMAP_OEA)
! 3550: __asm volatile("sync; mtsdr1 %0; isync"
! 3551: :: "r"((uintptr_t)pmap_pteg_table | (pmap_pteg_mask >> 10)));
! 3552: #elif defined (PMAP_OEA64) || defined (PMAP_OEA64_BRIDGE)
! 3553: __asm __volatile("sync; mtsdr1 %0; isync"
! 3554: :: "r"((uintptr_t)pmap_pteg_table |
! 3555: (32 - __builtin_clz(pmap_pteg_mask >> 11))));
1.41 matt 3556: #endif
1.99 ! thorpej 3557: tlbia();
1.91 macallan 3558:
3559: #if defined(PMAPDEBUG)
3560: if ( pmapdebug )
3561: pmap_print_mmuregs();
3562: #endif
1.1 matt 3563: }
1.99 ! thorpej 3564:
! 3565: /*
! 3566: * This is not part of the defined PMAP interface and is specific to the
! 3567: * PowerPC architecture. This is called during initppc, before the system
! 3568: * is really initialized.
! 3569: */
! 3570: void
! 3571: pmap_bootstrap(paddr_t kernelstart, paddr_t kernelend)
! 3572: {
! 3573: pmap_bootstrap1(kernelstart, kernelend);
! 3574: pmap_bootstrap2();
! 3575: }
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to pmap.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / powerpc / oea