Annotation of src/sys/arch/powerpc/oea/pmap.c, Revision 1.48.30.1
1.48.30.1! bouyer 1: /* $NetBSD$ */
1.1 matt 2: /*-
3: * Copyright (c) 2001 The NetBSD Foundation, Inc.
4: * All rights reserved.
5: *
6: * This code is derived from software contributed to The NetBSD Foundation
7: * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc.
8: *
1.38 sanjayl 9: * Support for PPC64 Bridge mode added by Sanjay Lal <sanjayl@kymasys.com>
10: * of Kyma Systems LLC.
11: *
1.1 matt 12: * Redistribution and use in source and binary forms, with or without
13: * modification, are permitted provided that the following conditions
14: * are met:
15: * 1. Redistributions of source code must retain the above copyright
16: * notice, this list of conditions and the following disclaimer.
17: * 2. Redistributions in binary form must reproduce the above copyright
18: * notice, this list of conditions and the following disclaimer in the
19: * documentation and/or other materials provided with the distribution.
20: * 3. All advertising materials mentioning features or use of this software
21: * must display the following acknowledgement:
22: * This product includes software developed by the NetBSD
23: * Foundation, Inc. and its contributors.
24: * 4. Neither the name of The NetBSD Foundation nor the names of its
25: * contributors may be used to endorse or promote products derived
26: * from this software without specific prior written permission.
27: *
28: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
29: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
30: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
31: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
32: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38: * POSSIBILITY OF SUCH DAMAGE.
39: */
40:
41: /*
42: * Copyright (C) 1995, 1996 Wolfgang Solfrank.
43: * Copyright (C) 1995, 1996 TooLs GmbH.
44: * All rights reserved.
45: *
46: * Redistribution and use in source and binary forms, with or without
47: * modification, are permitted provided that the following conditions
48: * are met:
49: * 1. Redistributions of source code must retain the above copyright
50: * notice, this list of conditions and the following disclaimer.
51: * 2. Redistributions in binary form must reproduce the above copyright
52: * notice, this list of conditions and the following disclaimer in the
53: * documentation and/or other materials provided with the distribution.
54: * 3. All advertising materials mentioning features or use of this software
55: * must display the following acknowledgement:
56: * This product includes software developed by TooLs GmbH.
57: * 4. The name of TooLs GmbH may not be used to endorse or promote products
58: * derived from this software without specific prior written permission.
59: *
60: * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
61: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
62: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
63: * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
64: * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
65: * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
66: * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
67: * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
68: * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
69: * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
70: */
1.11 lukem 71:
72: #include <sys/cdefs.h>
1.48.30.1! bouyer 73: __KERNEL_RCSID(0, "$NetBSD$");
1.1 matt 74:
1.18 matt 75: #include "opt_ppcarch.h"
1.1 matt 76: #include "opt_altivec.h"
77: #include "opt_pmap.h"
78: #include <sys/param.h>
79: #include <sys/malloc.h>
80: #include <sys/proc.h>
81: #include <sys/user.h>
82: #include <sys/pool.h>
83: #include <sys/queue.h>
84: #include <sys/device.h> /* for evcnt */
85: #include <sys/systm.h>
1.48.30.1! bouyer 86: #include <sys/atomic.h>
1.1 matt 87:
88: #if __NetBSD_Version__ < 105010000
89: #include <vm/vm.h>
90: #include <vm/vm_kern.h>
91: #define splvm() splimp()
92: #endif
93:
94: #include <uvm/uvm.h>
95:
96: #include <machine/pcb.h>
97: #include <machine/powerpc.h>
98: #include <powerpc/spr.h>
99: #include <powerpc/oea/sr_601.h>
100: #include <powerpc/bat.h>
1.38 sanjayl 101: #include <powerpc/stdarg.h>
1.1 matt 102:
103: #if defined(DEBUG) || defined(PMAPCHECK)
104: #define STATIC
105: #else
106: #define STATIC static
107: #endif
108:
109: #ifdef ALTIVEC
110: int pmap_use_altivec;
111: #endif
112:
1.2 matt 113: volatile struct pteg *pmap_pteg_table;
1.1 matt 114: unsigned int pmap_pteg_cnt;
115: unsigned int pmap_pteg_mask;
1.21 aymeric 116: #ifdef PMAP_MEMLIMIT
117: paddr_t pmap_memlimit = PMAP_MEMLIMIT;
118: #else
1.6 thorpej 119: paddr_t pmap_memlimit = -PAGE_SIZE; /* there is no limit */
1.21 aymeric 120: #endif
1.1 matt 121:
122: struct pmap kernel_pmap_;
123: unsigned int pmap_pages_stolen;
124: u_long pmap_pte_valid;
125: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
126: u_long pmap_pvo_enter_depth;
127: u_long pmap_pvo_remove_depth;
128: #endif
129:
130: int physmem;
131: #ifndef MSGBUFADDR
132: extern paddr_t msgbuf_paddr;
133: #endif
134:
135: static struct mem_region *mem, *avail;
136: static u_int mem_cnt, avail_cnt;
137:
138: #ifdef __HAVE_PMAP_PHYSSEG
139: /*
140: * This is a cache of referenced/modified bits.
141: * Bits herein are shifted by ATTRSHFT.
142: */
143: #define ATTR_SHFT 4
144: struct pmap_physseg pmap_physseg;
145: #endif
146:
147: /*
1.38 sanjayl 148: * The following structure is aligned to 32 bytes
1.1 matt 149: */
150: struct pvo_entry {
151: LIST_ENTRY(pvo_entry) pvo_vlink; /* Link to common virt page */
152: TAILQ_ENTRY(pvo_entry) pvo_olink; /* Link to overflow entry */
153: struct pte pvo_pte; /* Prebuilt PTE */
154: pmap_t pvo_pmap; /* ptr to owning pmap */
155: vaddr_t pvo_vaddr; /* VA of entry */
156: #define PVO_PTEGIDX_MASK 0x0007 /* which PTEG slot */
157: #define PVO_PTEGIDX_VALID 0x0008 /* slot is valid */
158: #define PVO_WIRED 0x0010 /* PVO entry is wired */
159: #define PVO_MANAGED 0x0020 /* PVO e. for managed page */
160: #define PVO_EXECUTABLE 0x0040 /* PVO e. for executable page */
1.39 matt 161: #define PVO_WIRED_P(pvo) ((pvo)->pvo_vaddr & PVO_WIRED)
162: #define PVO_MANAGED_P(pvo) ((pvo)->pvo_vaddr & PVO_MANAGED)
163: #define PVO_EXECUTABLE_P(pvo) ((pvo)->pvo_vaddr & PVO_EXECUTABLE)
1.12 matt 164: #define PVO_ENTER_INSERT 0 /* PVO has been removed */
165: #define PVO_SPILL_UNSET 1 /* PVO has been evicted */
166: #define PVO_SPILL_SET 2 /* PVO has been spilled */
167: #define PVO_SPILL_INSERT 3 /* PVO has been inserted */
168: #define PVO_PMAP_PAGE_PROTECT 4 /* PVO has changed */
169: #define PVO_PMAP_PROTECT 5 /* PVO has changed */
170: #define PVO_REMOVE 6 /* PVO has been removed */
171: #define PVO_WHERE_MASK 15
172: #define PVO_WHERE_SHFT 8
1.38 sanjayl 173: } __attribute__ ((aligned (32)));
1.1 matt 174: #define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF)
175: #define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
176: #define PVO_PTEGIDX_ISSET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID)
177: #define PVO_PTEGIDX_CLR(pvo) \
178: ((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK)))
179: #define PVO_PTEGIDX_SET(pvo,i) \
180: ((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID))
1.12 matt 181: #define PVO_WHERE(pvo,w) \
182: ((pvo)->pvo_vaddr &= ~(PVO_WHERE_MASK << PVO_WHERE_SHFT), \
183: (pvo)->pvo_vaddr |= ((PVO_ ## w) << PVO_WHERE_SHFT))
1.1 matt 184:
185: TAILQ_HEAD(pvo_tqhead, pvo_entry);
186: struct pvo_tqhead *pmap_pvo_table; /* pvo entries by ptegroup index */
187: struct pvo_head pmap_pvo_kunmanaged = LIST_HEAD_INITIALIZER(pmap_pvo_kunmanaged); /* list of unmanaged pages */
188: struct pvo_head pmap_pvo_unmanaged = LIST_HEAD_INITIALIZER(pmap_pvo_unmanaged); /* list of unmanaged pages */
189:
190: struct pool pmap_pool; /* pool for pmap structures */
191: struct pool pmap_upvo_pool; /* pool for pvo entries for unmanaged pages */
192: struct pool pmap_mpvo_pool; /* pool for pvo entries for managed pages */
193:
194: /*
195: * We keep a cache of unmanaged pages to be used for pvo entries for
196: * unmanaged pages.
197: */
198: struct pvo_page {
199: SIMPLEQ_ENTRY(pvo_page) pvop_link;
200: };
201: SIMPLEQ_HEAD(pvop_head, pvo_page);
202: struct pvop_head pmap_upvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_upvop_head);
203: struct pvop_head pmap_mpvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_mpvop_head);
204: u_long pmap_upvop_free;
205: u_long pmap_upvop_maxfree;
206: u_long pmap_mpvop_free;
207: u_long pmap_mpvop_maxfree;
208:
209: STATIC void *pmap_pool_ualloc(struct pool *, int);
210: STATIC void *pmap_pool_malloc(struct pool *, int);
211:
212: STATIC void pmap_pool_ufree(struct pool *, void *);
213: STATIC void pmap_pool_mfree(struct pool *, void *);
214:
215: static struct pool_allocator pmap_pool_mallocator = {
1.43 garbled 216: .pa_alloc = pmap_pool_malloc,
217: .pa_free = pmap_pool_mfree,
218: .pa_pagesz = 0,
1.1 matt 219: };
220:
221: static struct pool_allocator pmap_pool_uallocator = {
1.43 garbled 222: .pa_alloc = pmap_pool_ualloc,
223: .pa_free = pmap_pool_ufree,
224: .pa_pagesz = 0,
1.1 matt 225: };
226:
227: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
1.2 matt 228: void pmap_pte_print(volatile struct pte *);
1.1 matt 229: void pmap_pteg_check(void);
230: void pmap_pteg_dist(void);
231: void pmap_print_pte(pmap_t, vaddr_t);
232: void pmap_print_mmuregs(void);
233: #endif
234:
235: #if defined(DEBUG) || defined(PMAPCHECK)
236: #ifdef PMAPCHECK
237: int pmapcheck = 1;
238: #else
239: int pmapcheck = 0;
240: #endif
241: void pmap_pvo_verify(void);
242: STATIC void pmap_pvo_check(const struct pvo_entry *);
243: #define PMAP_PVO_CHECK(pvo) \
244: do { \
245: if (pmapcheck) \
246: pmap_pvo_check(pvo); \
247: } while (0)
248: #else
249: #define PMAP_PVO_CHECK(pvo) do { } while (/*CONSTCOND*/0)
250: #endif
1.2 matt 251: STATIC int pmap_pte_insert(int, struct pte *);
1.1 matt 252: STATIC int pmap_pvo_enter(pmap_t, struct pool *, struct pvo_head *,
1.2 matt 253: vaddr_t, paddr_t, register_t, int);
1.33 chs 254: STATIC void pmap_pvo_remove(struct pvo_entry *, int, struct pvo_head *);
255: STATIC void pmap_pvo_free(struct pvo_entry *);
256: STATIC void pmap_pvo_free_list(struct pvo_head *);
1.1 matt 257: STATIC struct pvo_entry *pmap_pvo_find_va(pmap_t, vaddr_t, int *);
1.2 matt 258: STATIC volatile struct pte *pmap_pvo_to_pte(const struct pvo_entry *, int);
1.25 chs 259: STATIC struct pvo_entry *pmap_pvo_reclaim(struct pmap *);
1.14 chs 260: STATIC void pvo_set_exec(struct pvo_entry *);
261: STATIC void pvo_clear_exec(struct pvo_entry *);
1.1 matt 262:
263: STATIC void tlbia(void);
264:
265: STATIC void pmap_release(pmap_t);
266: STATIC void *pmap_boot_find_memory(psize_t, psize_t, int);
267:
1.25 chs 268: static uint32_t pmap_pvo_reclaim_nextidx;
269: #ifdef DEBUG
270: static int pmap_pvo_reclaim_debugctr;
271: #endif
272:
1.1 matt 273: #define VSID_NBPW (sizeof(uint32_t) * 8)
274: static uint32_t pmap_vsid_bitmap[NPMAPS / VSID_NBPW];
275:
276: static int pmap_initialized;
277:
278: #if defined(DEBUG) || defined(PMAPDEBUG)
279: #define PMAPDEBUG_BOOT 0x0001
280: #define PMAPDEBUG_PTE 0x0002
281: #define PMAPDEBUG_EXEC 0x0008
282: #define PMAPDEBUG_PVOENTER 0x0010
283: #define PMAPDEBUG_PVOREMOVE 0x0020
284: #define PMAPDEBUG_ACTIVATE 0x0100
285: #define PMAPDEBUG_CREATE 0x0200
286: #define PMAPDEBUG_ENTER 0x1000
287: #define PMAPDEBUG_KENTER 0x2000
288: #define PMAPDEBUG_KREMOVE 0x4000
289: #define PMAPDEBUG_REMOVE 0x8000
1.38 sanjayl 290:
1.1 matt 291: unsigned int pmapdebug = 0;
1.38 sanjayl 292:
1.1 matt 293: # define DPRINTF(x) printf x
294: # define DPRINTFN(n, x) if (pmapdebug & PMAPDEBUG_ ## n) printf x
295: #else
296: # define DPRINTF(x)
297: # define DPRINTFN(n, x)
298: #endif
299:
300:
301: #ifdef PMAPCOUNTERS
302: #define PMAPCOUNT(ev) ((pmap_evcnt_ ## ev).ev_count++)
303: #define PMAPCOUNT2(ev) ((ev).ev_count++)
304:
305: struct evcnt pmap_evcnt_mappings =
306: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
307: "pmap", "pages mapped");
308: struct evcnt pmap_evcnt_unmappings =
309: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings,
310: "pmap", "pages unmapped");
311:
312: struct evcnt pmap_evcnt_kernel_mappings =
313: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
314: "pmap", "kernel pages mapped");
315: struct evcnt pmap_evcnt_kernel_unmappings =
316: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_kernel_mappings,
317: "pmap", "kernel pages unmapped");
318:
319: struct evcnt pmap_evcnt_mappings_replaced =
320: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
321: "pmap", "page mappings replaced");
322:
323: struct evcnt pmap_evcnt_exec_mappings =
324: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings,
325: "pmap", "exec pages mapped");
326: struct evcnt pmap_evcnt_exec_cached =
327: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings,
328: "pmap", "exec pages cached");
329:
330: struct evcnt pmap_evcnt_exec_synced =
331: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings,
332: "pmap", "exec pages synced");
333: struct evcnt pmap_evcnt_exec_synced_clear_modify =
334: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings,
335: "pmap", "exec pages synced (CM)");
1.37 matt 336: struct evcnt pmap_evcnt_exec_synced_pvo_remove =
337: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings,
338: "pmap", "exec pages synced (PR)");
1.1 matt 339:
340: struct evcnt pmap_evcnt_exec_uncached_page_protect =
341: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings,
342: "pmap", "exec pages uncached (PP)");
343: struct evcnt pmap_evcnt_exec_uncached_clear_modify =
344: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings,
345: "pmap", "exec pages uncached (CM)");
346: struct evcnt pmap_evcnt_exec_uncached_zero_page =
347: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings,
348: "pmap", "exec pages uncached (ZP)");
349: struct evcnt pmap_evcnt_exec_uncached_copy_page =
350: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings,
351: "pmap", "exec pages uncached (CP)");
1.37 matt 352: struct evcnt pmap_evcnt_exec_uncached_pvo_remove =
353: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings,
354: "pmap", "exec pages uncached (PR)");
1.1 matt 355:
356: struct evcnt pmap_evcnt_updates =
357: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
358: "pmap", "updates");
359: struct evcnt pmap_evcnt_collects =
360: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
361: "pmap", "collects");
362: struct evcnt pmap_evcnt_copies =
363: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
364: "pmap", "copies");
365:
366: struct evcnt pmap_evcnt_ptes_spilled =
367: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
368: "pmap", "ptes spilled from overflow");
369: struct evcnt pmap_evcnt_ptes_unspilled =
370: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
371: "pmap", "ptes not spilled");
372: struct evcnt pmap_evcnt_ptes_evicted =
373: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
374: "pmap", "ptes evicted");
375:
376: struct evcnt pmap_evcnt_ptes_primary[8] = {
377: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
378: "pmap", "ptes added at primary[0]"),
379: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
380: "pmap", "ptes added at primary[1]"),
381: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
382: "pmap", "ptes added at primary[2]"),
383: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
384: "pmap", "ptes added at primary[3]"),
385:
386: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
387: "pmap", "ptes added at primary[4]"),
388: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
389: "pmap", "ptes added at primary[5]"),
390: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
391: "pmap", "ptes added at primary[6]"),
392: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
393: "pmap", "ptes added at primary[7]"),
394: };
395: struct evcnt pmap_evcnt_ptes_secondary[8] = {
396: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
397: "pmap", "ptes added at secondary[0]"),
398: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
399: "pmap", "ptes added at secondary[1]"),
400: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
401: "pmap", "ptes added at secondary[2]"),
402: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
403: "pmap", "ptes added at secondary[3]"),
404:
405: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
406: "pmap", "ptes added at secondary[4]"),
407: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
408: "pmap", "ptes added at secondary[5]"),
409: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
410: "pmap", "ptes added at secondary[6]"),
411: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
412: "pmap", "ptes added at secondary[7]"),
413: };
414: struct evcnt pmap_evcnt_ptes_removed =
415: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
416: "pmap", "ptes removed");
417: struct evcnt pmap_evcnt_ptes_changed =
418: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
419: "pmap", "ptes changed");
1.26 matt 420: struct evcnt pmap_evcnt_pvos_reclaimed =
421: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
422: "pmap", "pvos reclaimed");
423: struct evcnt pmap_evcnt_pvos_failed =
424: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL,
425: "pmap", "pvo allocation failures");
1.1 matt 426:
427: /*
428: * From pmap_subr.c
429: */
430: extern struct evcnt pmap_evcnt_zeroed_pages;
431: extern struct evcnt pmap_evcnt_copied_pages;
432: extern struct evcnt pmap_evcnt_idlezeroed_pages;
1.26 matt 433:
434: EVCNT_ATTACH_STATIC(pmap_evcnt_mappings);
435: EVCNT_ATTACH_STATIC(pmap_evcnt_mappings_replaced);
436: EVCNT_ATTACH_STATIC(pmap_evcnt_unmappings);
437:
438: EVCNT_ATTACH_STATIC(pmap_evcnt_kernel_mappings);
439: EVCNT_ATTACH_STATIC(pmap_evcnt_kernel_unmappings);
440:
441: EVCNT_ATTACH_STATIC(pmap_evcnt_exec_mappings);
442: EVCNT_ATTACH_STATIC(pmap_evcnt_exec_cached);
443: EVCNT_ATTACH_STATIC(pmap_evcnt_exec_synced);
444: EVCNT_ATTACH_STATIC(pmap_evcnt_exec_synced_clear_modify);
1.37 matt 445: EVCNT_ATTACH_STATIC(pmap_evcnt_exec_synced_pvo_remove);
1.26 matt 446:
447: EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_page_protect);
448: EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_clear_modify);
449: EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_zero_page);
450: EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_copy_page);
1.37 matt 451: EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_pvo_remove);
1.26 matt 452:
453: EVCNT_ATTACH_STATIC(pmap_evcnt_zeroed_pages);
454: EVCNT_ATTACH_STATIC(pmap_evcnt_copied_pages);
455: EVCNT_ATTACH_STATIC(pmap_evcnt_idlezeroed_pages);
456:
457: EVCNT_ATTACH_STATIC(pmap_evcnt_updates);
458: EVCNT_ATTACH_STATIC(pmap_evcnt_collects);
459: EVCNT_ATTACH_STATIC(pmap_evcnt_copies);
460:
461: EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_spilled);
462: EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_unspilled);
463: EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_evicted);
464: EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_removed);
465: EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_changed);
466:
467: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 0);
468: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 1);
469: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 2);
470: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 3);
471: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 4);
472: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 5);
473: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 6);
474: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 7);
475: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 0);
476: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 1);
477: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 2);
478: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 3);
479: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 4);
480: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 5);
481: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 6);
482: EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 7);
483:
484: EVCNT_ATTACH_STATIC(pmap_evcnt_pvos_reclaimed);
485: EVCNT_ATTACH_STATIC(pmap_evcnt_pvos_failed);
1.1 matt 486: #else
487: #define PMAPCOUNT(ev) ((void) 0)
488: #define PMAPCOUNT2(ev) ((void) 0)
489: #endif
490:
1.35 perry 491: #define TLBIE(va) __asm volatile("tlbie %0" :: "r"(va))
1.38 sanjayl 492:
493: /* XXXSL: this needs to be moved to assembler */
494: #define TLBIEL(va) __asm __volatile("tlbie %0" :: "r"(va))
495:
1.35 perry 496: #define TLBSYNC() __asm volatile("tlbsync")
497: #define SYNC() __asm volatile("sync")
498: #define EIEIO() __asm volatile("eieio")
1.1 matt 499: #define MFMSR() mfmsr()
500: #define MTMSR(psl) mtmsr(psl)
501: #define MFPVR() mfpvr()
502: #define MFSRIN(va) mfsrin(va)
503: #define MFTB() mfrtcltbl()
504:
1.38 sanjayl 505: #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE)
1.35 perry 506: static inline register_t
1.1 matt 507: mfsrin(vaddr_t va)
508: {
1.2 matt 509: register_t sr;
1.35 perry 510: __asm volatile ("mfsrin %0,%1" : "=r"(sr) : "r"(va));
1.1 matt 511: return sr;
512: }
1.38 sanjayl 513: #endif /* PPC_OEA*/
514:
515: #if defined (PPC_OEA64_BRIDGE)
516: extern void mfmsr64 (register64_t *result);
517: #endif /* PPC_OEA64_BRIDGE */
518:
1.48.30.1! bouyer 519: #define PMAP_LOCK() KERNEL_LOCK(1, NULL)
! 520: #define PMAP_UNLOCK() KERNEL_UNLOCK_ONE(NULL)
1.1 matt 521:
1.35 perry 522: static inline register_t
1.1 matt 523: pmap_interrupts_off(void)
524: {
1.2 matt 525: register_t msr = MFMSR();
1.1 matt 526: if (msr & PSL_EE)
527: MTMSR(msr & ~PSL_EE);
528: return msr;
529: }
530:
531: static void
1.2 matt 532: pmap_interrupts_restore(register_t msr)
1.1 matt 533: {
534: if (msr & PSL_EE)
535: MTMSR(msr);
536: }
537:
1.35 perry 538: static inline u_int32_t
1.1 matt 539: mfrtcltbl(void)
540: {
541:
542: if ((MFPVR() >> 16) == MPC601)
543: return (mfrtcl() >> 7);
544: else
545: return (mftbl());
546: }
547:
548: /*
549: * These small routines may have to be replaced,
550: * if/when we support processors other that the 604.
551: */
552:
553: void
554: tlbia(void)
555: {
1.47 macallan 556: char *i;
1.1 matt 557:
558: SYNC();
1.38 sanjayl 559: #if defined(PPC_OEA)
1.1 matt 560: /*
561: * Why not use "tlbia"? Because not all processors implement it.
562: *
1.20 wiz 563: * This needs to be a per-CPU callback to do the appropriate thing
1.1 matt 564: * for the CPU. XXX
565: */
1.47 macallan 566: for (i = 0; i < (char *)0x00040000; i += 0x00001000) {
1.1 matt 567: TLBIE(i);
568: EIEIO();
569: SYNC();
570: }
1.38 sanjayl 571: #elif defined (PPC_OEA64) || defined (PPC_OEA64_BRIDGE)
572: printf("Invalidating ALL TLB entries......\n");
573: /* This is specifically for the 970, 970UM v1.6 pp. 140. */
1.46 christos 574: for (i = 0; i <= (void *)0xFF000; i += 0x00001000) {
1.38 sanjayl 575: TLBIEL(i);
576: EIEIO();
577: SYNC();
578: }
579: #endif
1.1 matt 580: TLBSYNC();
581: SYNC();
582: }
583:
1.35 perry 584: static inline register_t
1.2 matt 585: va_to_vsid(const struct pmap *pm, vaddr_t addr)
1.1 matt 586: {
1.38 sanjayl 587: #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE)
588: return (pm->pm_sr[addr >> ADDR_SR_SHFT] & SR_VSID) >> SR_VSID_SHFT;
589: #else /* PPC_OEA64 */
1.18 matt 590: #if 0
591: const struct ste *ste;
592: register_t hash;
593: int i;
594:
595: hash = (addr >> ADDR_ESID_SHFT) & ADDR_ESID_HASH;
596:
597: /*
598: * Try the primary group first
599: */
600: ste = pm->pm_stes[hash].stes;
601: for (i = 0; i < 8; i++, ste++) {
602: if (ste->ste_hi & STE_V) &&
603: (addr & ~(ADDR_POFF|ADDR_PIDX)) == (ste->ste_hi & STE_ESID))
604: return ste;
605: }
606:
607: /*
608: * Then the secondary group.
609: */
610: ste = pm->pm_stes[hash ^ ADDR_ESID_HASH].stes;
611: for (i = 0; i < 8; i++, ste++) {
612: if (ste->ste_hi & STE_V) &&
613: (addr & ~(ADDR_POFF|ADDR_PIDX)) == (ste->ste_hi & STE_ESID))
614: return addr;
615: }
616:
617: return NULL;
618: #else
619: /*
620: * Rather than searching the STE groups for the VSID, we know
621: * how we generate that from the ESID and so do that.
622: */
623: return VSID_MAKE(addr >> ADDR_SR_SHFT, pm->pm_vsid) >> SR_VSID_SHFT;
624: #endif
1.38 sanjayl 625: #endif /* PPC_OEA */
1.1 matt 626: }
627:
1.35 perry 628: static inline register_t
1.2 matt 629: va_to_pteg(const struct pmap *pm, vaddr_t addr)
1.1 matt 630: {
1.2 matt 631: register_t hash;
632:
633: hash = va_to_vsid(pm, addr) ^ ((addr & ADDR_PIDX) >> ADDR_PIDX_SHFT);
1.1 matt 634: return hash & pmap_pteg_mask;
635: }
636:
637: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
638: /*
639: * Given a PTE in the page table, calculate the VADDR that hashes to it.
640: * The only bit of magic is that the top 4 bits of the address doesn't
641: * technically exist in the PTE. But we know we reserved 4 bits of the
642: * VSID for it so that's how we get it.
643: */
644: static vaddr_t
1.2 matt 645: pmap_pte_to_va(volatile const struct pte *pt)
1.1 matt 646: {
647: vaddr_t va;
648: uintptr_t ptaddr = (uintptr_t) pt;
649:
650: if (pt->pte_hi & PTE_HID)
1.2 matt 651: ptaddr ^= (pmap_pteg_mask * sizeof(struct pteg));
1.1 matt 652:
1.18 matt 653: /* PPC Bits 10-19 PPC64 Bits 42-51 */
1.38 sanjayl 654: #if defined(PPC_OEA)
1.4 matt 655: va = ((pt->pte_hi >> PTE_VSID_SHFT) ^ (ptaddr / sizeof(struct pteg))) & 0x3ff;
1.38 sanjayl 656: #elif defined (PPC_OEA64) || defined (PPC_OEA64_BRIDGE)
657: va = ((pt->pte_hi >> PTE_VSID_SHFT) ^ (ptaddr / sizeof(struct pteg))) & 0x7ff;
658: #endif
1.1 matt 659: va <<= ADDR_PIDX_SHFT;
660:
1.18 matt 661: /* PPC Bits 4-9 PPC64 Bits 36-41 */
1.1 matt 662: va |= (pt->pte_hi & PTE_API) << ADDR_API_SHFT;
663:
1.38 sanjayl 664: #if defined(PPC_OEA64)
1.18 matt 665: /* PPC63 Bits 0-35 */
666: /* va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT; */
1.38 sanjayl 667: #elif defined(PPC_OEA) || defined(PPC_OEA64_BRIDGE)
1.1 matt 668: /* PPC Bits 0-3 */
669: va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT;
1.18 matt 670: #endif
1.1 matt 671:
672: return va;
673: }
674: #endif
675:
1.35 perry 676: static inline struct pvo_head *
1.1 matt 677: pa_to_pvoh(paddr_t pa, struct vm_page **pg_p)
678: {
679: #ifdef __HAVE_VM_PAGE_MD
680: struct vm_page *pg;
681:
682: pg = PHYS_TO_VM_PAGE(pa);
683: if (pg_p != NULL)
684: *pg_p = pg;
685: if (pg == NULL)
686: return &pmap_pvo_unmanaged;
687: return &pg->mdpage.mdpg_pvoh;
688: #endif
689: #ifdef __HAVE_PMAP_PHYSSEG
690: int bank, pg;
691:
692: bank = vm_physseg_find(atop(pa), &pg);
693: if (pg_p != NULL)
694: *pg_p = pg;
695: if (bank == -1)
696: return &pmap_pvo_unmanaged;
697: return &vm_physmem[bank].pmseg.pvoh[pg];
698: #endif
699: }
700:
1.35 perry 701: static inline struct pvo_head *
1.1 matt 702: vm_page_to_pvoh(struct vm_page *pg)
703: {
704: #ifdef __HAVE_VM_PAGE_MD
705: return &pg->mdpage.mdpg_pvoh;
706: #endif
707: #ifdef __HAVE_PMAP_PHYSSEG
708: return pa_to_pvoh(VM_PAGE_TO_PHYS(pg), NULL);
709: #endif
710: }
711:
712:
713: #ifdef __HAVE_PMAP_PHYSSEG
1.35 perry 714: static inline char *
1.1 matt 715: pa_to_attr(paddr_t pa)
716: {
717: int bank, pg;
718:
719: bank = vm_physseg_find(atop(pa), &pg);
720: if (bank == -1)
721: return NULL;
722: return &vm_physmem[bank].pmseg.attrs[pg];
723: }
724: #endif
725:
1.35 perry 726: static inline void
1.1 matt 727: pmap_attr_clear(struct vm_page *pg, int ptebit)
728: {
729: #ifdef __HAVE_PMAP_PHYSSEG
730: *pa_to_attr(VM_PAGE_TO_PHYS(pg)) &= ~(ptebit >> ATTR_SHFT);
731: #endif
732: #ifdef __HAVE_VM_PAGE_MD
733: pg->mdpage.mdpg_attrs &= ~ptebit;
734: #endif
735: }
736:
1.35 perry 737: static inline int
1.1 matt 738: pmap_attr_fetch(struct vm_page *pg)
739: {
740: #ifdef __HAVE_PMAP_PHYSSEG
741: return *pa_to_attr(VM_PAGE_TO_PHYS(pg)) << ATTR_SHFT;
742: #endif
743: #ifdef __HAVE_VM_PAGE_MD
744: return pg->mdpage.mdpg_attrs;
745: #endif
746: }
747:
1.35 perry 748: static inline void
1.1 matt 749: pmap_attr_save(struct vm_page *pg, int ptebit)
750: {
751: #ifdef __HAVE_PMAP_PHYSSEG
752: *pa_to_attr(VM_PAGE_TO_PHYS(pg)) |= (ptebit >> ATTR_SHFT);
753: #endif
754: #ifdef __HAVE_VM_PAGE_MD
755: pg->mdpage.mdpg_attrs |= ptebit;
756: #endif
757: }
758:
1.35 perry 759: static inline int
1.2 matt 760: pmap_pte_compare(const volatile struct pte *pt, const struct pte *pvo_pt)
1.1 matt 761: {
762: if (pt->pte_hi == pvo_pt->pte_hi
763: #if 0
764: && ((pt->pte_lo ^ pvo_pt->pte_lo) &
765: ~(PTE_REF|PTE_CHG)) == 0
766: #endif
767: )
768: return 1;
769: return 0;
770: }
771:
1.35 perry 772: static inline void
1.2 matt 773: pmap_pte_create(struct pte *pt, const struct pmap *pm, vaddr_t va, register_t pte_lo)
1.1 matt 774: {
775: /*
776: * Construct the PTE. Default to IMB initially. Valid bit
777: * only gets set when the real pte is set in memory.
778: *
779: * Note: Don't set the valid bit for correct operation of tlb update.
780: */
1.38 sanjayl 781: #if defined(PPC_OEA)
1.2 matt 782: pt->pte_hi = (va_to_vsid(pm, va) << PTE_VSID_SHFT)
783: | (((va & ADDR_PIDX) >> (ADDR_API_SHFT - PTE_API_SHFT)) & PTE_API);
1.1 matt 784: pt->pte_lo = pte_lo;
1.38 sanjayl 785: #elif defined (PPC_OEA64_BRIDGE)
786: pt->pte_hi = ((u_int64_t)va_to_vsid(pm, va) << PTE_VSID_SHFT)
787: | (((va & ADDR_PIDX) >> (ADDR_API_SHFT - PTE_API_SHFT)) & PTE_API);
788: pt->pte_lo = (u_int64_t) pte_lo;
789: #elif defined (PPC_OEA64)
790: #error PPC_OEA64 not supported
791: #endif /* PPC_OEA */
1.1 matt 792: }
793:
1.35 perry 794: static inline void
1.2 matt 795: pmap_pte_synch(volatile struct pte *pt, struct pte *pvo_pt)
1.1 matt 796: {
797: pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF|PTE_CHG);
798: }
799:
1.35 perry 800: static inline void
1.2 matt 801: pmap_pte_clear(volatile struct pte *pt, vaddr_t va, int ptebit)
1.1 matt 802: {
803: /*
804: * As shown in Section 7.6.3.2.3
805: */
806: pt->pte_lo &= ~ptebit;
807: TLBIE(va);
808: SYNC();
809: EIEIO();
810: TLBSYNC();
811: SYNC();
812: }
813:
1.35 perry 814: static inline void
1.2 matt 815: pmap_pte_set(volatile struct pte *pt, struct pte *pvo_pt)
1.1 matt 816: {
817: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
818: if (pvo_pt->pte_hi & PTE_VALID)
819: panic("pte_set: setting an already valid pte %p", pvo_pt);
820: #endif
821: pvo_pt->pte_hi |= PTE_VALID;
1.38 sanjayl 822:
1.1 matt 823: /*
824: * Update the PTE as defined in section 7.6.3.1
825: * Note that the REF/CHG bits are from pvo_pt and thus should
826: * have been saved so this routine can restore them (if desired).
827: */
828: pt->pte_lo = pvo_pt->pte_lo;
829: EIEIO();
830: pt->pte_hi = pvo_pt->pte_hi;
1.38 sanjayl 831: TLBSYNC();
1.1 matt 832: SYNC();
833: pmap_pte_valid++;
834: }
835:
1.35 perry 836: static inline void
1.2 matt 837: pmap_pte_unset(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va)
1.1 matt 838: {
839: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
840: if ((pvo_pt->pte_hi & PTE_VALID) == 0)
841: panic("pte_unset: attempt to unset an inactive pte#1 %p/%p", pvo_pt, pt);
842: if ((pt->pte_hi & PTE_VALID) == 0)
843: panic("pte_unset: attempt to unset an inactive pte#2 %p/%p", pvo_pt, pt);
844: #endif
845:
846: pvo_pt->pte_hi &= ~PTE_VALID;
847: /*
848: * Force the ref & chg bits back into the PTEs.
849: */
850: SYNC();
851: /*
852: * Invalidate the pte ... (Section 7.6.3.3)
853: */
854: pt->pte_hi &= ~PTE_VALID;
855: SYNC();
856: TLBIE(va);
857: SYNC();
858: EIEIO();
859: TLBSYNC();
860: SYNC();
861: /*
862: * Save the ref & chg bits ...
863: */
864: pmap_pte_synch(pt, pvo_pt);
865: pmap_pte_valid--;
866: }
867:
1.35 perry 868: static inline void
1.2 matt 869: pmap_pte_change(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va)
1.1 matt 870: {
871: /*
872: * Invalidate the PTE
873: */
874: pmap_pte_unset(pt, pvo_pt, va);
875: pmap_pte_set(pt, pvo_pt);
876: }
877:
878: /*
879: * Try to insert the PTE @ *pvo_pt into the pmap_pteg_table at ptegidx
880: * (either primary or secondary location).
881: *
882: * Note: both the destination and source PTEs must not have PTE_VALID set.
883: */
884:
885: STATIC int
1.2 matt 886: pmap_pte_insert(int ptegidx, struct pte *pvo_pt)
1.1 matt 887: {
1.2 matt 888: volatile struct pte *pt;
1.1 matt 889: int i;
890:
891: #if defined(DEBUG)
1.38 sanjayl 892: #if defined (PPC_OEA)
1.18 matt 893: DPRINTFN(PTE, ("pmap_pte_insert: idx 0x%x, pte 0x%x 0x%x\n",
1.19 mjl 894: ptegidx, (unsigned int) pvo_pt->pte_hi, (unsigned int) pvo_pt->pte_lo));
1.38 sanjayl 895: #elif defined (PPC_OEA64_BRIDGE)
896: DPRINTFN(PTE, ("pmap_pte_insert: idx 0x%x, pte 0x%016llx 0x%016llx\n",
897: ptegidx, (unsigned long long) pvo_pt->pte_hi,
898: (unsigned long long) pvo_pt->pte_lo));
899:
900: #endif
1.1 matt 901: #endif
902: /*
903: * First try primary hash.
904: */
905: for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
906: if ((pt->pte_hi & PTE_VALID) == 0) {
907: pvo_pt->pte_hi &= ~PTE_HID;
908: pmap_pte_set(pt, pvo_pt);
909: return i;
910: }
911: }
912:
913: /*
914: * Now try secondary hash.
915: */
916: ptegidx ^= pmap_pteg_mask;
917: for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
918: if ((pt->pte_hi & PTE_VALID) == 0) {
919: pvo_pt->pte_hi |= PTE_HID;
920: pmap_pte_set(pt, pvo_pt);
921: return i;
922: }
923: }
924: return -1;
925: }
926:
927: /*
928: * Spill handler.
929: *
930: * Tries to spill a page table entry from the overflow area.
931: * This runs in either real mode (if dealing with a exception spill)
932: * or virtual mode when dealing with manually spilling one of the
933: * kernel's pte entries. In either case, interrupts are already
934: * disabled.
935: */
1.14 chs 936:
1.1 matt 937: int
1.44 thorpej 938: pmap_pte_spill(struct pmap *pm, vaddr_t addr, bool exec)
1.1 matt 939: {
940: struct pvo_entry *source_pvo, *victim_pvo, *next_pvo;
941: struct pvo_entry *pvo;
1.15 dyoung 942: /* XXX: gcc -- vpvoh is always set at either *1* or *2* */
943: struct pvo_tqhead *pvoh, *vpvoh = NULL;
1.1 matt 944: int ptegidx, i, j;
1.2 matt 945: volatile struct pteg *pteg;
946: volatile struct pte *pt;
1.1 matt 947:
1.48.30.1! bouyer 948: PMAP_LOCK();
! 949:
1.2 matt 950: ptegidx = va_to_pteg(pm, addr);
1.1 matt 951:
952: /*
953: * Have to substitute some entry. Use the primary hash for this.
1.12 matt 954: * Use low bits of timebase as random generator. Make sure we are
955: * not picking a kernel pte for replacement.
1.1 matt 956: */
957: pteg = &pmap_pteg_table[ptegidx];
958: i = MFTB() & 7;
1.12 matt 959: for (j = 0; j < 8; j++) {
960: pt = &pteg->pt[i];
961: if ((pt->pte_hi & PTE_VALID) == 0 ||
962: VSID_TO_HASH((pt->pte_hi & PTE_VSID) >> PTE_VSID_SHFT)
963: != KERNEL_VSIDBITS)
964: break;
965: i = (i + 1) & 7;
966: }
967: KASSERT(j < 8);
1.1 matt 968:
969: source_pvo = NULL;
970: victim_pvo = NULL;
971: pvoh = &pmap_pvo_table[ptegidx];
972: TAILQ_FOREACH(pvo, pvoh, pvo_olink) {
973:
974: /*
975: * We need to find pvo entry for this address...
976: */
977: PMAP_PVO_CHECK(pvo); /* sanity check */
978:
979: /*
980: * If we haven't found the source and we come to a PVO with
981: * a valid PTE, then we know we can't find it because all
982: * evicted PVOs always are first in the list.
983: */
984: if (source_pvo == NULL && (pvo->pvo_pte.pte_hi & PTE_VALID))
985: break;
1.2 matt 986: if (source_pvo == NULL && pm == pvo->pvo_pmap &&
987: addr == PVO_VADDR(pvo)) {
1.1 matt 988:
989: /*
990: * Now we have found the entry to be spilled into the
991: * pteg. Attempt to insert it into the page table.
992: */
993: j = pmap_pte_insert(ptegidx, &pvo->pvo_pte);
994: if (j >= 0) {
995: PVO_PTEGIDX_SET(pvo, j);
996: PMAP_PVO_CHECK(pvo); /* sanity check */
1.12 matt 997: PVO_WHERE(pvo, SPILL_INSERT);
1.1 matt 998: pvo->pvo_pmap->pm_evictions--;
999: PMAPCOUNT(ptes_spilled);
1000: PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID)
1001: ? pmap_evcnt_ptes_secondary
1002: : pmap_evcnt_ptes_primary)[j]);
1003:
1004: /*
1005: * Since we keep the evicted entries at the
1006: * from of the PVO list, we need move this
1007: * (now resident) PVO after the evicted
1008: * entries.
1009: */
1010: next_pvo = TAILQ_NEXT(pvo, pvo_olink);
1011:
1012: /*
1.5 matt 1013: * If we don't have to move (either we were the
1014: * last entry or the next entry was valid),
1.1 matt 1015: * don't change our position. Otherwise
1016: * move ourselves to the tail of the queue.
1017: */
1018: if (next_pvo != NULL &&
1019: !(next_pvo->pvo_pte.pte_hi & PTE_VALID)) {
1020: TAILQ_REMOVE(pvoh, pvo, pvo_olink);
1021: TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink);
1022: }
1.48.30.1! bouyer 1023: PMAP_UNLOCK();
1.1 matt 1024: return 1;
1025: }
1026: source_pvo = pvo;
1.39 matt 1027: if (exec && !PVO_EXECUTABLE_P(source_pvo)) {
1.14 chs 1028: return 0;
1029: }
1.1 matt 1030: if (victim_pvo != NULL)
1031: break;
1032: }
1033:
1034: /*
1035: * We also need the pvo entry of the victim we are replacing
1036: * so save the R & C bits of the PTE.
1037: */
1038: if ((pt->pte_hi & PTE_HID) == 0 && victim_pvo == NULL &&
1039: pmap_pte_compare(pt, &pvo->pvo_pte)) {
1.15 dyoung 1040: vpvoh = pvoh; /* *1* */
1.1 matt 1041: victim_pvo = pvo;
1042: if (source_pvo != NULL)
1043: break;
1044: }
1045: }
1046:
1047: if (source_pvo == NULL) {
1048: PMAPCOUNT(ptes_unspilled);
1.48.30.1! bouyer 1049: PMAP_UNLOCK();
1.1 matt 1050: return 0;
1051: }
1052:
1053: if (victim_pvo == NULL) {
1054: if ((pt->pte_hi & PTE_HID) == 0)
1055: panic("pmap_pte_spill: victim p-pte (%p) has "
1056: "no pvo entry!", pt);
1057:
1058: /*
1059: * If this is a secondary PTE, we need to search
1060: * its primary pvo bucket for the matching PVO.
1061: */
1.15 dyoung 1062: vpvoh = &pmap_pvo_table[ptegidx ^ pmap_pteg_mask]; /* *2* */
1.1 matt 1063: TAILQ_FOREACH(pvo, vpvoh, pvo_olink) {
1064: PMAP_PVO_CHECK(pvo); /* sanity check */
1065:
1066: /*
1067: * We also need the pvo entry of the victim we are
1068: * replacing so save the R & C bits of the PTE.
1069: */
1070: if (pmap_pte_compare(pt, &pvo->pvo_pte)) {
1071: victim_pvo = pvo;
1072: break;
1073: }
1074: }
1075: if (victim_pvo == NULL)
1076: panic("pmap_pte_spill: victim s-pte (%p) has "
1077: "no pvo entry!", pt);
1078: }
1079:
1080: /*
1.12 matt 1081: * The victim should be not be a kernel PVO/PTE entry.
1082: */
1083: KASSERT(victim_pvo->pvo_pmap != pmap_kernel());
1084: KASSERT(PVO_PTEGIDX_ISSET(victim_pvo));
1085: KASSERT(PVO_PTEGIDX_GET(victim_pvo) == i);
1086:
1087: /*
1.1 matt 1088: * We are invalidating the TLB entry for the EA for the
1089: * we are replacing even though its valid; If we don't
1090: * we lose any ref/chg bit changes contained in the TLB
1091: * entry.
1092: */
1093: source_pvo->pvo_pte.pte_hi &= ~PTE_HID;
1094:
1095: /*
1096: * To enforce the PVO list ordering constraint that all
1097: * evicted entries should come before all valid entries,
1098: * move the source PVO to the tail of its list and the
1099: * victim PVO to the head of its list (which might not be
1100: * the same list, if the victim was using the secondary hash).
1101: */
1102: TAILQ_REMOVE(pvoh, source_pvo, pvo_olink);
1103: TAILQ_INSERT_TAIL(pvoh, source_pvo, pvo_olink);
1104: TAILQ_REMOVE(vpvoh, victim_pvo, pvo_olink);
1105: TAILQ_INSERT_HEAD(vpvoh, victim_pvo, pvo_olink);
1106: pmap_pte_unset(pt, &victim_pvo->pvo_pte, victim_pvo->pvo_vaddr);
1107: pmap_pte_set(pt, &source_pvo->pvo_pte);
1108: victim_pvo->pvo_pmap->pm_evictions++;
1109: source_pvo->pvo_pmap->pm_evictions--;
1.12 matt 1110: PVO_WHERE(victim_pvo, SPILL_UNSET);
1111: PVO_WHERE(source_pvo, SPILL_SET);
1.1 matt 1112:
1113: PVO_PTEGIDX_CLR(victim_pvo);
1114: PVO_PTEGIDX_SET(source_pvo, i);
1115: PMAPCOUNT2(pmap_evcnt_ptes_primary[i]);
1116: PMAPCOUNT(ptes_spilled);
1117: PMAPCOUNT(ptes_evicted);
1118: PMAPCOUNT(ptes_removed);
1119:
1120: PMAP_PVO_CHECK(victim_pvo);
1121: PMAP_PVO_CHECK(source_pvo);
1.48.30.1! bouyer 1122:
! 1123: PMAP_UNLOCK();
1.1 matt 1124: return 1;
1125: }
1126:
1127: /*
1128: * Restrict given range to physical memory
1129: */
1130: void
1131: pmap_real_memory(paddr_t *start, psize_t *size)
1132: {
1133: struct mem_region *mp;
1134:
1135: for (mp = mem; mp->size; mp++) {
1136: if (*start + *size > mp->start
1137: && *start < mp->start + mp->size) {
1138: if (*start < mp->start) {
1139: *size -= mp->start - *start;
1140: *start = mp->start;
1141: }
1142: if (*start + *size > mp->start + mp->size)
1143: *size = mp->start + mp->size - *start;
1144: return;
1145: }
1146: }
1147: *size = 0;
1148: }
1149:
1150: /*
1151: * Initialize anything else for pmap handling.
1152: * Called during vm_init().
1153: */
1154: void
1155: pmap_init(void)
1156: {
1157: #ifdef __HAVE_PMAP_PHYSSEG
1158: struct pvo_tqhead *pvoh;
1159: int bank;
1160: long sz;
1161: char *attr;
1162:
1163: pvoh = pmap_physseg.pvoh;
1164: attr = pmap_physseg.attrs;
1165: for (bank = 0; bank < vm_nphysseg; bank++) {
1166: sz = vm_physmem[bank].end - vm_physmem[bank].start;
1167: vm_physmem[bank].pmseg.pvoh = pvoh;
1168: vm_physmem[bank].pmseg.attrs = attr;
1169: for (; sz > 0; sz--, pvoh++, attr++) {
1170: TAILQ_INIT(pvoh);
1171: *attr = 0;
1172: }
1173: }
1174: #endif
1175:
1176: pool_init(&pmap_mpvo_pool, sizeof(struct pvo_entry),
1177: sizeof(struct pvo_entry), 0, 0, "pmap_mpvopl",
1.48 ad 1178: &pmap_pool_mallocator, IPL_NONE);
1.1 matt 1179:
1180: pool_setlowat(&pmap_mpvo_pool, 1008);
1181:
1182: pmap_initialized = 1;
1183:
1184: }
1185:
1186: /*
1.10 thorpej 1187: * How much virtual space does the kernel get?
1188: */
1189: void
1190: pmap_virtual_space(vaddr_t *start, vaddr_t *end)
1191: {
1192: /*
1193: * For now, reserve one segment (minus some overhead) for kernel
1194: * virtual memory
1195: */
1196: *start = VM_MIN_KERNEL_ADDRESS;
1197: *end = VM_MAX_KERNEL_ADDRESS;
1198: }
1199:
1200: /*
1.1 matt 1201: * Allocate, initialize, and return a new physical map.
1202: */
1203: pmap_t
1204: pmap_create(void)
1205: {
1206: pmap_t pm;
1.38 sanjayl 1207:
1.1 matt 1208: pm = pool_get(&pmap_pool, PR_WAITOK);
1.46 christos 1209: memset((void *)pm, 0, sizeof *pm);
1.1 matt 1210: pmap_pinit(pm);
1211:
1212: DPRINTFN(CREATE,("pmap_create: pm %p:\n"
1.18 matt 1213: "\t%06x %06x %06x %06x %06x %06x %06x %06x\n"
1214: "\t%06x %06x %06x %06x %06x %06x %06x %06x\n", pm,
1.19 mjl 1215: (unsigned int) pm->pm_sr[0], (unsigned int) pm->pm_sr[1],
1216: (unsigned int) pm->pm_sr[2], (unsigned int) pm->pm_sr[3],
1217: (unsigned int) pm->pm_sr[4], (unsigned int) pm->pm_sr[5],
1218: (unsigned int) pm->pm_sr[6], (unsigned int) pm->pm_sr[7],
1219: (unsigned int) pm->pm_sr[8], (unsigned int) pm->pm_sr[9],
1220: (unsigned int) pm->pm_sr[10], (unsigned int) pm->pm_sr[11],
1221: (unsigned int) pm->pm_sr[12], (unsigned int) pm->pm_sr[13],
1222: (unsigned int) pm->pm_sr[14], (unsigned int) pm->pm_sr[15]));
1.1 matt 1223: return pm;
1224: }
1225:
1226: /*
1227: * Initialize a preallocated and zeroed pmap structure.
1228: */
1229: void
1230: pmap_pinit(pmap_t pm)
1231: {
1.2 matt 1232: register_t entropy = MFTB();
1233: register_t mask;
1234: int i;
1.1 matt 1235:
1236: /*
1237: * Allocate some segment registers for this pmap.
1238: */
1239: pm->pm_refs = 1;
1.48.30.1! bouyer 1240: PMAP_LOCK();
1.2 matt 1241: for (i = 0; i < NPMAPS; i += VSID_NBPW) {
1242: static register_t pmap_vsidcontext;
1243: register_t hash;
1244: unsigned int n;
1.1 matt 1245:
1246: /* Create a new value by multiplying by a prime adding in
1247: * entropy from the timebase register. This is to make the
1248: * VSID more random so that the PT Hash function collides
1249: * less often. (note that the prime causes gcc to do shifts
1250: * instead of a multiply)
1251: */
1252: pmap_vsidcontext = (pmap_vsidcontext * 0x1105) + entropy;
1253: hash = pmap_vsidcontext & (NPMAPS - 1);
1.23 aymeric 1254: if (hash == 0) { /* 0 is special, avoid it */
1255: entropy += 0xbadf00d;
1.1 matt 1256: continue;
1.23 aymeric 1257: }
1.1 matt 1258: n = hash >> 5;
1.2 matt 1259: mask = 1L << (hash & (VSID_NBPW-1));
1260: hash = pmap_vsidcontext;
1.1 matt 1261: if (pmap_vsid_bitmap[n] & mask) { /* collision? */
1262: /* anything free in this bucket? */
1.2 matt 1263: if (~pmap_vsid_bitmap[n] == 0) {
1.23 aymeric 1264: entropy = hash ^ (hash >> 16);
1.1 matt 1265: continue;
1266: }
1267: i = ffs(~pmap_vsid_bitmap[n]) - 1;
1.2 matt 1268: mask = 1L << i;
1269: hash &= ~(VSID_NBPW-1);
1.1 matt 1270: hash |= i;
1271: }
1.18 matt 1272: hash &= PTE_VSID >> PTE_VSID_SHFT;
1.1 matt 1273: pmap_vsid_bitmap[n] |= mask;
1.18 matt 1274: pm->pm_vsid = hash;
1.38 sanjayl 1275: #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE)
1.1 matt 1276: for (i = 0; i < 16; i++)
1.14 chs 1277: pm->pm_sr[i] = VSID_MAKE(i, hash) | SR_PRKEY |
1278: SR_NOEXEC;
1.18 matt 1279: #endif
1.48.30.1! bouyer 1280: PMAP_UNLOCK();
1.1 matt 1281: return;
1282: }
1.48.30.1! bouyer 1283: PMAP_UNLOCK();
1.1 matt 1284: panic("pmap_pinit: out of segments");
1285: }
1286:
1287: /*
1288: * Add a reference to the given pmap.
1289: */
1290: void
1291: pmap_reference(pmap_t pm)
1292: {
1.48.30.1! bouyer 1293: atomic_inc_uint(&pm->pm_refs);
1.1 matt 1294: }
1295:
1296: /*
1297: * Retire the given pmap from service.
1298: * Should only be called if the map contains no valid mappings.
1299: */
1300: void
1301: pmap_destroy(pmap_t pm)
1302: {
1.48.30.1! bouyer 1303: if (atomic_dec_uint_nv(&pm->pm_refs) == 0) {
1.1 matt 1304: pmap_release(pm);
1305: pool_put(&pmap_pool, pm);
1306: }
1307: }
1308:
1309: /*
1310: * Release any resources held by the given physical map.
1311: * Called when a pmap initialized by pmap_pinit is being released.
1312: */
1313: void
1314: pmap_release(pmap_t pm)
1315: {
1316: int idx, mask;
1.39 matt 1317:
1318: KASSERT(pm->pm_stats.resident_count == 0);
1319: KASSERT(pm->pm_stats.wired_count == 0);
1.1 matt 1320:
1.48.30.1! bouyer 1321: PMAP_LOCK();
1.1 matt 1322: if (pm->pm_sr[0] == 0)
1323: panic("pmap_release");
1.22 aymeric 1324: idx = pm->pm_vsid & (NPMAPS-1);
1.1 matt 1325: mask = 1 << (idx % VSID_NBPW);
1326: idx /= VSID_NBPW;
1.22 aymeric 1327:
1328: KASSERT(pmap_vsid_bitmap[idx] & mask);
1.1 matt 1329: pmap_vsid_bitmap[idx] &= ~mask;
1.48.30.1! bouyer 1330: PMAP_UNLOCK();
1.1 matt 1331: }
1332:
1333: /*
1334: * Copy the range specified by src_addr/len
1335: * from the source map to the range dst_addr/len
1336: * in the destination map.
1337: *
1338: * This routine is only advisory and need not do anything.
1339: */
1340: void
1341: pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr,
1342: vsize_t len, vaddr_t src_addr)
1343: {
1344: PMAPCOUNT(copies);
1345: }
1346:
1347: /*
1348: * Require that all active physical maps contain no
1349: * incorrect entries NOW.
1350: */
1351: void
1352: pmap_update(struct pmap *pmap)
1353: {
1354: PMAPCOUNT(updates);
1355: TLBSYNC();
1356: }
1357:
1358: /*
1359: * Garbage collects the physical map system for
1360: * pages which are no longer used.
1361: * Success need not be guaranteed -- that is, there
1362: * may well be pages which are not referenced, but
1363: * others may be collected.
1364: * Called by the pageout daemon when pages are scarce.
1365: */
1366: void
1367: pmap_collect(pmap_t pm)
1368: {
1369: PMAPCOUNT(collects);
1370: }
1371:
1.35 perry 1372: static inline int
1.1 matt 1373: pmap_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx)
1374: {
1375: int pteidx;
1376: /*
1377: * We can find the actual pte entry without searching by
1378: * grabbing the PTEG index from 3 unused bits in pte_lo[11:9]
1379: * and by noticing the HID bit.
1380: */
1381: pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo);
1382: if (pvo->pvo_pte.pte_hi & PTE_HID)
1383: pteidx ^= pmap_pteg_mask * 8;
1384: return pteidx;
1385: }
1386:
1.2 matt 1387: volatile struct pte *
1.1 matt 1388: pmap_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
1389: {
1.2 matt 1390: volatile struct pte *pt;
1.1 matt 1391:
1392: #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK)
1393: if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0)
1394: return NULL;
1395: #endif
1396:
1397: /*
1398: * If we haven't been supplied the ptegidx, calculate it.
1399: */
1400: if (pteidx == -1) {
1401: int ptegidx;
1.2 matt 1402: ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr);
1.1 matt 1403: pteidx = pmap_pvo_pte_index(pvo, ptegidx);
1404: }
1405:
1406: pt = &pmap_pteg_table[pteidx >> 3].pt[pteidx & 7];
1407:
1408: #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK)
1409: return pt;
1410: #else
1411: if ((pvo->pvo_pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) {
1412: panic("pmap_pvo_to_pte: pvo %p: has valid pte in "
1413: "pvo but no valid pte index", pvo);
1414: }
1415: if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) {
1416: panic("pmap_pvo_to_pte: pvo %p: has valid pte index in "
1417: "pvo but no valid pte", pvo);
1418: }
1419:
1420: if ((pt->pte_hi ^ (pvo->pvo_pte.pte_hi & ~PTE_VALID)) == PTE_VALID) {
1421: if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) {
1422: #if defined(DEBUG) || defined(PMAPCHECK)
1423: pmap_pte_print(pt);
1424: #endif
1425: panic("pmap_pvo_to_pte: pvo %p: has valid pte in "
1426: "pmap_pteg_table %p but invalid in pvo",
1427: pvo, pt);
1428: }
1429: if (((pt->pte_lo ^ pvo->pvo_pte.pte_lo) & ~(PTE_CHG|PTE_REF)) != 0) {
1430: #if defined(DEBUG) || defined(PMAPCHECK)
1431: pmap_pte_print(pt);
1432: #endif
1433: panic("pmap_pvo_to_pte: pvo %p: pvo pte does "
1434: "not match pte %p in pmap_pteg_table",
1435: pvo, pt);
1436: }
1437: return pt;
1438: }
1439:
1440: if (pvo->pvo_pte.pte_hi & PTE_VALID) {
1441: #if defined(DEBUG) || defined(PMAPCHECK)
1442: pmap_pte_print(pt);
1443: #endif
1.12 matt 1444: panic("pmap_pvo_to_pte: pvo %p: has nomatching pte %p in "
1.1 matt 1445: "pmap_pteg_table but valid in pvo", pvo, pt);
1446: }
1447: return NULL;
1448: #endif /* !(!DIAGNOSTIC && !DEBUG && !PMAPCHECK) */
1449: }
1450:
1451: struct pvo_entry *
1452: pmap_pvo_find_va(pmap_t pm, vaddr_t va, int *pteidx_p)
1453: {
1454: struct pvo_entry *pvo;
1455: int ptegidx;
1456:
1457: va &= ~ADDR_POFF;
1.2 matt 1458: ptegidx = va_to_pteg(pm, va);
1.1 matt 1459:
1460: TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
1461: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1462: if ((uintptr_t) pvo >= SEGMENT_LENGTH)
1463: panic("pmap_pvo_find_va: invalid pvo %p on "
1464: "list %#x (%p)", pvo, ptegidx,
1465: &pmap_pvo_table[ptegidx]);
1466: #endif
1467: if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
1468: if (pteidx_p)
1469: *pteidx_p = pmap_pvo_pte_index(pvo, ptegidx);
1470: return pvo;
1471: }
1472: }
1.38 sanjayl 1473: if ((pm == pmap_kernel()) && (va < SEGMENT_LENGTH))
1.48.30.1! bouyer 1474: panic("%s: returning NULL for %s pmap, va: 0x%08lx\n", __func__,
1.38 sanjayl 1475: (pm == pmap_kernel() ? "kernel" : "user"), va);
1.1 matt 1476: return NULL;
1477: }
1478:
1479: #if defined(DEBUG) || defined(PMAPCHECK)
1480: void
1481: pmap_pvo_check(const struct pvo_entry *pvo)
1482: {
1483: struct pvo_head *pvo_head;
1484: struct pvo_entry *pvo0;
1.2 matt 1485: volatile struct pte *pt;
1.1 matt 1486: int failed = 0;
1487:
1.48.30.1! bouyer 1488: PMAP_LOCK();
! 1489:
1.1 matt 1490: if ((uintptr_t)(pvo+1) >= SEGMENT_LENGTH)
1491: panic("pmap_pvo_check: pvo %p: invalid address", pvo);
1492:
1493: if ((uintptr_t)(pvo->pvo_pmap+1) >= SEGMENT_LENGTH) {
1494: printf("pmap_pvo_check: pvo %p: invalid pmap address %p\n",
1495: pvo, pvo->pvo_pmap);
1496: failed = 1;
1497: }
1498:
1499: if ((uintptr_t)TAILQ_NEXT(pvo, pvo_olink) >= SEGMENT_LENGTH ||
1500: (((uintptr_t)TAILQ_NEXT(pvo, pvo_olink)) & 0x1f) != 0) {
1501: printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n",
1502: pvo, TAILQ_NEXT(pvo, pvo_olink));
1503: failed = 1;
1504: }
1505:
1506: if ((uintptr_t)LIST_NEXT(pvo, pvo_vlink) >= SEGMENT_LENGTH ||
1507: (((uintptr_t)LIST_NEXT(pvo, pvo_vlink)) & 0x1f) != 0) {
1508: printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n",
1509: pvo, LIST_NEXT(pvo, pvo_vlink));
1510: failed = 1;
1511: }
1512:
1.39 matt 1513: if (PVO_MANAGED_P(pvo)) {
1.1 matt 1514: pvo_head = pa_to_pvoh(pvo->pvo_pte.pte_lo & PTE_RPGN, NULL);
1515: } else {
1516: if (pvo->pvo_vaddr < VM_MIN_KERNEL_ADDRESS) {
1517: printf("pmap_pvo_check: pvo %p: non kernel address "
1518: "on kernel unmanaged list\n", pvo);
1519: failed = 1;
1520: }
1521: pvo_head = &pmap_pvo_kunmanaged;
1522: }
1523: LIST_FOREACH(pvo0, pvo_head, pvo_vlink) {
1524: if (pvo0 == pvo)
1525: break;
1526: }
1527: if (pvo0 == NULL) {
1528: printf("pmap_pvo_check: pvo %p: not present "
1529: "on its vlist head %p\n", pvo, pvo_head);
1530: failed = 1;
1531: }
1532: if (pvo != pmap_pvo_find_va(pvo->pvo_pmap, pvo->pvo_vaddr, NULL)) {
1533: printf("pmap_pvo_check: pvo %p: not present "
1534: "on its olist head\n", pvo);
1535: failed = 1;
1536: }
1537: pt = pmap_pvo_to_pte(pvo, -1);
1538: if (pt == NULL) {
1539: if (pvo->pvo_pte.pte_hi & PTE_VALID) {
1540: printf("pmap_pvo_check: pvo %p: pte_hi VALID but "
1541: "no PTE\n", pvo);
1542: failed = 1;
1543: }
1544: } else {
1545: if ((uintptr_t) pt < (uintptr_t) &pmap_pteg_table[0] ||
1546: (uintptr_t) pt >=
1547: (uintptr_t) &pmap_pteg_table[pmap_pteg_cnt]) {
1548: printf("pmap_pvo_check: pvo %p: pte %p not in "
1549: "pteg table\n", pvo, pt);
1550: failed = 1;
1551: }
1552: if (((((uintptr_t) pt) >> 3) & 7) != PVO_PTEGIDX_GET(pvo)) {
1553: printf("pmap_pvo_check: pvo %p: pte_hi VALID but "
1554: "no PTE\n", pvo);
1555: failed = 1;
1556: }
1557: if (pvo->pvo_pte.pte_hi != pt->pte_hi) {
1558: printf("pmap_pvo_check: pvo %p: pte_hi differ: "
1.19 mjl 1559: "%#x/%#x\n", pvo, (unsigned int) pvo->pvo_pte.pte_hi, (unsigned int) pt->pte_hi);
1.1 matt 1560: failed = 1;
1561: }
1562: if (((pvo->pvo_pte.pte_lo ^ pt->pte_lo) &
1563: (PTE_PP|PTE_WIMG|PTE_RPGN)) != 0) {
1564: printf("pmap_pvo_check: pvo %p: pte_lo differ: "
1.18 matt 1565: "%#x/%#x\n", pvo,
1.19 mjl 1566: (unsigned int) (pvo->pvo_pte.pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN)),
1567: (unsigned int) (pt->pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN)));
1.1 matt 1568: failed = 1;
1569: }
1570: if ((pmap_pte_to_va(pt) ^ PVO_VADDR(pvo)) & 0x0fffffff) {
1571: printf("pmap_pvo_check: pvo %p: PTE %p derived VA %#lx"
1572: " doesn't not match PVO's VA %#lx\n",
1573: pvo, pt, pmap_pte_to_va(pt), PVO_VADDR(pvo));
1574: failed = 1;
1575: }
1576: if (failed)
1577: pmap_pte_print(pt);
1578: }
1579: if (failed)
1580: panic("pmap_pvo_check: pvo %p, pm %p: bugcheck!", pvo,
1581: pvo->pvo_pmap);
1.48.30.1! bouyer 1582:
! 1583: PMAP_UNLOCK();
1.1 matt 1584: }
1585: #endif /* DEBUG || PMAPCHECK */
1586:
1587: /*
1.25 chs 1588: * Search the PVO table looking for a non-wired entry.
1589: * If we find one, remove it and return it.
1590: */
1591:
1592: struct pvo_entry *
1593: pmap_pvo_reclaim(struct pmap *pm)
1594: {
1595: struct pvo_tqhead *pvoh;
1596: struct pvo_entry *pvo;
1597: uint32_t idx, endidx;
1598:
1599: endidx = pmap_pvo_reclaim_nextidx;
1600: for (idx = (endidx + 1) & pmap_pteg_mask; idx != endidx;
1601: idx = (idx + 1) & pmap_pteg_mask) {
1602: pvoh = &pmap_pvo_table[idx];
1603: TAILQ_FOREACH(pvo, pvoh, pvo_olink) {
1.39 matt 1604: if (!PVO_WIRED_P(pvo)) {
1.33 chs 1605: pmap_pvo_remove(pvo, -1, NULL);
1.25 chs 1606: pmap_pvo_reclaim_nextidx = idx;
1.26 matt 1607: PMAPCOUNT(pvos_reclaimed);
1.25 chs 1608: return pvo;
1609: }
1610: }
1611: }
1612: return NULL;
1613: }
1614:
1615: /*
1.1 matt 1616: * This returns whether this is the first mapping of a page.
1617: */
1618: int
1619: pmap_pvo_enter(pmap_t pm, struct pool *pl, struct pvo_head *pvo_head,
1.2 matt 1620: vaddr_t va, paddr_t pa, register_t pte_lo, int flags)
1.1 matt 1621: {
1622: struct pvo_entry *pvo;
1623: struct pvo_tqhead *pvoh;
1.2 matt 1624: register_t msr;
1.1 matt 1625: int ptegidx;
1626: int i;
1627: int poolflags = PR_NOWAIT;
1628:
1.28 chs 1629: /*
1630: * Compute the PTE Group index.
1631: */
1632: va &= ~ADDR_POFF;
1633: ptegidx = va_to_pteg(pm, va);
1634:
1635: msr = pmap_interrupts_off();
1636:
1.1 matt 1637: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1638: if (pmap_pvo_remove_depth > 0)
1639: panic("pmap_pvo_enter: called while pmap_pvo_remove active!");
1640: if (++pmap_pvo_enter_depth > 1)
1641: panic("pmap_pvo_enter: called recursively!");
1642: #endif
1643:
1644: /*
1645: * Remove any existing mapping for this page. Reuse the
1646: * pvo entry if there a mapping.
1647: */
1648: TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
1649: if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
1650: #ifdef DEBUG
1651: if ((pmapdebug & PMAPDEBUG_PVOENTER) &&
1652: ((pvo->pvo_pte.pte_lo ^ (pa|pte_lo)) &
1653: ~(PTE_REF|PTE_CHG)) == 0 &&
1654: va < VM_MIN_KERNEL_ADDRESS) {
1.18 matt 1655: printf("pmap_pvo_enter: pvo %p: dup %#x/%#lx\n",
1.19 mjl 1656: pvo, (unsigned int) pvo->pvo_pte.pte_lo, (unsigned int) pte_lo|pa);
1.18 matt 1657: printf("pmap_pvo_enter: pte_hi=%#x sr=%#x\n",
1.19 mjl 1658: (unsigned int) pvo->pvo_pte.pte_hi,
1659: (unsigned int) pm->pm_sr[va >> ADDR_SR_SHFT]);
1.1 matt 1660: pmap_pte_print(pmap_pvo_to_pte(pvo, -1));
1661: #ifdef DDBX
1662: Debugger();
1663: #endif
1664: }
1665: #endif
1666: PMAPCOUNT(mappings_replaced);
1.33 chs 1667: pmap_pvo_remove(pvo, -1, NULL);
1.1 matt 1668: break;
1669: }
1670: }
1671:
1672: /*
1673: * If we aren't overwriting an mapping, try to allocate
1674: */
1.26 matt 1675: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1676: --pmap_pvo_enter_depth;
1677: #endif
1.1 matt 1678: pmap_interrupts_restore(msr);
1.33 chs 1679: if (pvo) {
1680: pmap_pvo_free(pvo);
1681: }
1.1 matt 1682: pvo = pool_get(pl, poolflags);
1.25 chs 1683:
1684: #ifdef DEBUG
1685: /*
1686: * Exercise pmap_pvo_reclaim() a little.
1687: */
1688: if (pvo && (flags & PMAP_CANFAIL) != 0 &&
1689: pmap_pvo_reclaim_debugctr++ > 0x1000 &&
1690: (pmap_pvo_reclaim_debugctr & 0xff) == 0) {
1691: pool_put(pl, pvo);
1692: pvo = NULL;
1693: }
1694: #endif
1695:
1.1 matt 1696: msr = pmap_interrupts_off();
1.26 matt 1697: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1698: ++pmap_pvo_enter_depth;
1699: #endif
1.1 matt 1700: if (pvo == NULL) {
1701: pvo = pmap_pvo_reclaim(pm);
1702: if (pvo == NULL) {
1703: if ((flags & PMAP_CANFAIL) == 0)
1704: panic("pmap_pvo_enter: failed");
1705: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1706: pmap_pvo_enter_depth--;
1707: #endif
1.26 matt 1708: PMAPCOUNT(pvos_failed);
1.1 matt 1709: pmap_interrupts_restore(msr);
1710: return ENOMEM;
1711: }
1712: }
1.25 chs 1713:
1.1 matt 1714: pvo->pvo_vaddr = va;
1715: pvo->pvo_pmap = pm;
1716: pvo->pvo_vaddr &= ~ADDR_POFF;
1717: if (flags & VM_PROT_EXECUTE) {
1718: PMAPCOUNT(exec_mappings);
1.14 chs 1719: pvo_set_exec(pvo);
1.1 matt 1720: }
1721: if (flags & PMAP_WIRED)
1722: pvo->pvo_vaddr |= PVO_WIRED;
1723: if (pvo_head != &pmap_pvo_kunmanaged) {
1724: pvo->pvo_vaddr |= PVO_MANAGED;
1725: PMAPCOUNT(mappings);
1726: } else {
1727: PMAPCOUNT(kernel_mappings);
1728: }
1.2 matt 1729: pmap_pte_create(&pvo->pvo_pte, pm, va, pa | pte_lo);
1.1 matt 1730:
1731: LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink);
1.39 matt 1732: if (PVO_WIRED_P(pvo))
1.1 matt 1733: pvo->pvo_pmap->pm_stats.wired_count++;
1734: pvo->pvo_pmap->pm_stats.resident_count++;
1735: #if defined(DEBUG)
1.38 sanjayl 1736: /* if (pm != pmap_kernel() && va < VM_MIN_KERNEL_ADDRESS) */
1.1 matt 1737: DPRINTFN(PVOENTER,
1738: ("pmap_pvo_enter: pvo %p: pm %p va %#lx pa %#lx\n",
1739: pvo, pm, va, pa));
1740: #endif
1741:
1742: /*
1743: * We hope this succeeds but it isn't required.
1744: */
1745: pvoh = &pmap_pvo_table[ptegidx];
1746: i = pmap_pte_insert(ptegidx, &pvo->pvo_pte);
1747: if (i >= 0) {
1748: PVO_PTEGIDX_SET(pvo, i);
1.12 matt 1749: PVO_WHERE(pvo, ENTER_INSERT);
1.1 matt 1750: PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID)
1751: ? pmap_evcnt_ptes_secondary : pmap_evcnt_ptes_primary)[i]);
1752: TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink);
1.38 sanjayl 1753:
1.1 matt 1754: } else {
1755: /*
1756: * Since we didn't have room for this entry (which makes it
1757: * and evicted entry), place it at the head of the list.
1758: */
1759: TAILQ_INSERT_HEAD(pvoh, pvo, pvo_olink);
1760: PMAPCOUNT(ptes_evicted);
1761: pm->pm_evictions++;
1.12 matt 1762: /*
1763: * If this is a kernel page, make sure it's active.
1764: */
1765: if (pm == pmap_kernel()) {
1.45 thorpej 1766: i = pmap_pte_spill(pm, va, false);
1.12 matt 1767: KASSERT(i);
1768: }
1.1 matt 1769: }
1770: PMAP_PVO_CHECK(pvo); /* sanity check */
1771: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1772: pmap_pvo_enter_depth--;
1773: #endif
1774: pmap_interrupts_restore(msr);
1775: return 0;
1776: }
1777:
1778: void
1.33 chs 1779: pmap_pvo_remove(struct pvo_entry *pvo, int pteidx, struct pvo_head *pvol)
1.1 matt 1780: {
1.2 matt 1781: volatile struct pte *pt;
1.1 matt 1782: int ptegidx;
1783:
1784: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1785: if (++pmap_pvo_remove_depth > 1)
1786: panic("pmap_pvo_remove: called recursively!");
1787: #endif
1788:
1789: /*
1790: * If we haven't been supplied the ptegidx, calculate it.
1791: */
1792: if (pteidx == -1) {
1.2 matt 1793: ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr);
1.1 matt 1794: pteidx = pmap_pvo_pte_index(pvo, ptegidx);
1795: } else {
1796: ptegidx = pteidx >> 3;
1797: if (pvo->pvo_pte.pte_hi & PTE_HID)
1798: ptegidx ^= pmap_pteg_mask;
1799: }
1800: PMAP_PVO_CHECK(pvo); /* sanity check */
1801:
1802: /*
1803: * If there is an active pte entry, we need to deactivate it
1804: * (and save the ref & chg bits).
1805: */
1806: pt = pmap_pvo_to_pte(pvo, pteidx);
1807: if (pt != NULL) {
1808: pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
1.12 matt 1809: PVO_WHERE(pvo, REMOVE);
1.1 matt 1810: PVO_PTEGIDX_CLR(pvo);
1811: PMAPCOUNT(ptes_removed);
1812: } else {
1813: KASSERT(pvo->pvo_pmap->pm_evictions > 0);
1814: pvo->pvo_pmap->pm_evictions--;
1815: }
1816:
1817: /*
1.14 chs 1818: * Account for executable mappings.
1819: */
1.39 matt 1820: if (PVO_EXECUTABLE_P(pvo))
1.14 chs 1821: pvo_clear_exec(pvo);
1822:
1823: /*
1824: * Update our statistics.
1.1 matt 1825: */
1826: pvo->pvo_pmap->pm_stats.resident_count--;
1.39 matt 1827: if (PVO_WIRED_P(pvo))
1.1 matt 1828: pvo->pvo_pmap->pm_stats.wired_count--;
1829:
1830: /*
1831: * Save the REF/CHG bits into their cache if the page is managed.
1832: */
1.39 matt 1833: if (PVO_MANAGED_P(pvo)) {
1.2 matt 1834: register_t ptelo = pvo->pvo_pte.pte_lo;
1.1 matt 1835: struct vm_page *pg = PHYS_TO_VM_PAGE(ptelo & PTE_RPGN);
1836:
1837: if (pg != NULL) {
1.37 matt 1838: /*
1839: * If this page was changed and it is mapped exec,
1840: * invalidate it.
1841: */
1842: if ((ptelo & PTE_CHG) &&
1843: (pmap_attr_fetch(pg) & PTE_EXEC)) {
1844: struct pvo_head *pvoh = vm_page_to_pvoh(pg);
1845: if (LIST_EMPTY(pvoh)) {
1846: DPRINTFN(EXEC, ("[pmap_pvo_remove: "
1847: "%#lx: clear-exec]\n",
1848: VM_PAGE_TO_PHYS(pg)));
1849: pmap_attr_clear(pg, PTE_EXEC);
1850: PMAPCOUNT(exec_uncached_pvo_remove);
1851: } else {
1852: DPRINTFN(EXEC, ("[pmap_pvo_remove: "
1853: "%#lx: syncicache]\n",
1854: VM_PAGE_TO_PHYS(pg)));
1855: pmap_syncicache(VM_PAGE_TO_PHYS(pg),
1856: PAGE_SIZE);
1857: PMAPCOUNT(exec_synced_pvo_remove);
1858: }
1859: }
1860:
1.1 matt 1861: pmap_attr_save(pg, ptelo & (PTE_REF|PTE_CHG));
1862: }
1863: PMAPCOUNT(unmappings);
1864: } else {
1865: PMAPCOUNT(kernel_unmappings);
1866: }
1867:
1868: /*
1869: * Remove the PVO from its lists and return it to the pool.
1870: */
1871: LIST_REMOVE(pvo, pvo_vlink);
1872: TAILQ_REMOVE(&pmap_pvo_table[ptegidx], pvo, pvo_olink);
1.33 chs 1873: if (pvol) {
1874: LIST_INSERT_HEAD(pvol, pvo, pvo_vlink);
1.25 chs 1875: }
1.1 matt 1876: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
1877: pmap_pvo_remove_depth--;
1878: #endif
1879: }
1880:
1.33 chs 1881: void
1882: pmap_pvo_free(struct pvo_entry *pvo)
1883: {
1884:
1.39 matt 1885: pool_put(PVO_MANAGED_P(pvo) ? &pmap_mpvo_pool : &pmap_upvo_pool, pvo);
1.33 chs 1886: }
1887:
1888: void
1889: pmap_pvo_free_list(struct pvo_head *pvol)
1890: {
1891: struct pvo_entry *pvo, *npvo;
1892:
1893: for (pvo = LIST_FIRST(pvol); pvo != NULL; pvo = npvo) {
1894: npvo = LIST_NEXT(pvo, pvo_vlink);
1895: LIST_REMOVE(pvo, pvo_vlink);
1896: pmap_pvo_free(pvo);
1897: }
1898: }
1899:
1.1 matt 1900: /*
1.14 chs 1901: * Mark a mapping as executable.
1902: * If this is the first executable mapping in the segment,
1903: * clear the noexec flag.
1904: */
1905: STATIC void
1906: pvo_set_exec(struct pvo_entry *pvo)
1907: {
1908: struct pmap *pm = pvo->pvo_pmap;
1909:
1.39 matt 1910: if (pm == pmap_kernel() || PVO_EXECUTABLE_P(pvo)) {
1.14 chs 1911: return;
1912: }
1913: pvo->pvo_vaddr |= PVO_EXECUTABLE;
1.38 sanjayl 1914: #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE)
1.18 matt 1915: {
1916: int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT;
1917: if (pm->pm_exec[sr]++ == 0) {
1918: pm->pm_sr[sr] &= ~SR_NOEXEC;
1919: }
1.14 chs 1920: }
1.18 matt 1921: #endif
1.14 chs 1922: }
1923:
1924: /*
1925: * Mark a mapping as non-executable.
1926: * If this was the last executable mapping in the segment,
1927: * set the noexec flag.
1928: */
1929: STATIC void
1930: pvo_clear_exec(struct pvo_entry *pvo)
1931: {
1932: struct pmap *pm = pvo->pvo_pmap;
1933:
1.39 matt 1934: if (pm == pmap_kernel() || !PVO_EXECUTABLE_P(pvo)) {
1.14 chs 1935: return;
1936: }
1937: pvo->pvo_vaddr &= ~PVO_EXECUTABLE;
1.38 sanjayl 1938: #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE)
1.18 matt 1939: {
1940: int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT;
1941: if (--pm->pm_exec[sr] == 0) {
1942: pm->pm_sr[sr] |= SR_NOEXEC;
1943: }
1.14 chs 1944: }
1.18 matt 1945: #endif
1.14 chs 1946: }
1947:
1948: /*
1.1 matt 1949: * Insert physical page at pa into the given pmap at virtual address va.
1950: */
1951: int
1952: pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags)
1953: {
1954: struct mem_region *mp;
1955: struct pvo_head *pvo_head;
1956: struct vm_page *pg;
1957: struct pool *pl;
1.2 matt 1958: register_t pte_lo;
1.1 matt 1959: int error;
1960: u_int pvo_flags;
1961: u_int was_exec = 0;
1962:
1.48.30.1! bouyer 1963: PMAP_LOCK();
! 1964:
1.1 matt 1965: if (__predict_false(!pmap_initialized)) {
1966: pvo_head = &pmap_pvo_kunmanaged;
1967: pl = &pmap_upvo_pool;
1968: pvo_flags = 0;
1969: pg = NULL;
1970: was_exec = PTE_EXEC;
1971: } else {
1972: pvo_head = pa_to_pvoh(pa, &pg);
1973: pl = &pmap_mpvo_pool;
1974: pvo_flags = PVO_MANAGED;
1975: }
1976:
1977: DPRINTFN(ENTER,
1978: ("pmap_enter(%p, 0x%lx, 0x%lx, 0x%x, 0x%x):",
1979: pm, va, pa, prot, flags));
1980:
1981: /*
1982: * If this is a managed page, and it's the first reference to the
1983: * page clear the execness of the page. Otherwise fetch the execness.
1984: */
1985: if (pg != NULL)
1986: was_exec = pmap_attr_fetch(pg) & PTE_EXEC;
1987:
1988: DPRINTFN(ENTER, (" was_exec=%d", was_exec));
1989:
1990: /*
1991: * Assume the page is cache inhibited and access is guarded unless
1992: * it's in our available memory array. If it is in the memory array,
1993: * asssume it's in memory coherent memory.
1994: */
1995: pte_lo = PTE_IG;
1996: if ((flags & PMAP_NC) == 0) {
1997: for (mp = mem; mp->size; mp++) {
1998: if (pa >= mp->start && pa < mp->start + mp->size) {
1999: pte_lo = PTE_M;
2000: break;
2001: }
2002: }
2003: }
2004:
2005: if (prot & VM_PROT_WRITE)
2006: pte_lo |= PTE_BW;
2007: else
2008: pte_lo |= PTE_BR;
2009:
2010: /*
2011: * If this was in response to a fault, "pre-fault" the PTE's
2012: * changed/referenced bit appropriately.
2013: */
2014: if (flags & VM_PROT_WRITE)
2015: pte_lo |= PTE_CHG;
1.30 chs 2016: if (flags & VM_PROT_ALL)
1.1 matt 2017: pte_lo |= PTE_REF;
2018:
2019: /*
2020: * We need to know if this page can be executable
2021: */
2022: flags |= (prot & VM_PROT_EXECUTE);
2023:
2024: /*
2025: * Record mapping for later back-translation and pte spilling.
2026: * This will overwrite any existing mapping.
2027: */
2028: error = pmap_pvo_enter(pm, pl, pvo_head, va, pa, pte_lo, flags);
2029:
2030: /*
2031: * Flush the real page from the instruction cache if this page is
2032: * mapped executable and cacheable and has not been flushed since
2033: * the last time it was modified.
2034: */
2035: if (error == 0 &&
2036: (flags & VM_PROT_EXECUTE) &&
2037: (pte_lo & PTE_I) == 0 &&
2038: was_exec == 0) {
2039: DPRINTFN(ENTER, (" syncicache"));
2040: PMAPCOUNT(exec_synced);
1.6 thorpej 2041: pmap_syncicache(pa, PAGE_SIZE);
1.1 matt 2042: if (pg != NULL) {
2043: pmap_attr_save(pg, PTE_EXEC);
2044: PMAPCOUNT(exec_cached);
2045: #if defined(DEBUG) || defined(PMAPDEBUG)
2046: if (pmapdebug & PMAPDEBUG_ENTER)
2047: printf(" marked-as-exec");
2048: else if (pmapdebug & PMAPDEBUG_EXEC)
2049: printf("[pmap_enter: %#lx: marked-as-exec]\n",
1.34 yamt 2050: VM_PAGE_TO_PHYS(pg));
1.1 matt 2051:
2052: #endif
2053: }
2054: }
2055:
2056: DPRINTFN(ENTER, (": error=%d\n", error));
2057:
1.48.30.1! bouyer 2058: PMAP_UNLOCK();
! 2059:
1.1 matt 2060: return error;
2061: }
2062:
2063: void
2064: pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot)
2065: {
2066: struct mem_region *mp;
1.2 matt 2067: register_t pte_lo;
1.1 matt 2068: int error;
2069:
1.38 sanjayl 2070: #if defined (PPC_OEA64_BRIDGE)
1.1 matt 2071: if (va < VM_MIN_KERNEL_ADDRESS)
2072: panic("pmap_kenter_pa: attempt to enter "
2073: "non-kernel address %#lx!", va);
1.38 sanjayl 2074: #endif
1.1 matt 2075:
2076: DPRINTFN(KENTER,
2077: ("pmap_kenter_pa(%#lx,%#lx,%#x)\n", va, pa, prot));
2078:
1.48.30.1! bouyer 2079: PMAP_LOCK();
! 2080:
1.1 matt 2081: /*
2082: * Assume the page is cache inhibited and access is guarded unless
2083: * it's in our available memory array. If it is in the memory array,
2084: * asssume it's in memory coherent memory.
2085: */
2086: pte_lo = PTE_IG;
1.4 matt 2087: if ((prot & PMAP_NC) == 0) {
2088: for (mp = mem; mp->size; mp++) {
2089: if (pa >= mp->start && pa < mp->start + mp->size) {
2090: pte_lo = PTE_M;
2091: break;
2092: }
1.1 matt 2093: }
2094: }
2095:
2096: if (prot & VM_PROT_WRITE)
2097: pte_lo |= PTE_BW;
2098: else
2099: pte_lo |= PTE_BR;
2100:
2101: /*
2102: * We don't care about REF/CHG on PVOs on the unmanaged list.
2103: */
2104: error = pmap_pvo_enter(pmap_kernel(), &pmap_upvo_pool,
2105: &pmap_pvo_kunmanaged, va, pa, pte_lo, prot|PMAP_WIRED);
2106:
2107: if (error != 0)
2108: panic("pmap_kenter_pa: failed to enter va %#lx pa %#lx: %d",
2109: va, pa, error);
1.48.30.1! bouyer 2110:
! 2111: PMAP_UNLOCK();
1.1 matt 2112: }
2113:
2114: void
2115: pmap_kremove(vaddr_t va, vsize_t len)
2116: {
2117: if (va < VM_MIN_KERNEL_ADDRESS)
2118: panic("pmap_kremove: attempt to remove "
2119: "non-kernel address %#lx!", va);
2120:
2121: DPRINTFN(KREMOVE,("pmap_kremove(%#lx,%#lx)\n", va, len));
2122: pmap_remove(pmap_kernel(), va, va + len);
2123: }
2124:
2125: /*
2126: * Remove the given range of mapping entries.
2127: */
2128: void
2129: pmap_remove(pmap_t pm, vaddr_t va, vaddr_t endva)
2130: {
1.33 chs 2131: struct pvo_head pvol;
1.1 matt 2132: struct pvo_entry *pvo;
1.2 matt 2133: register_t msr;
1.1 matt 2134: int pteidx;
2135:
1.48.30.1! bouyer 2136: PMAP_LOCK();
1.33 chs 2137: LIST_INIT(&pvol);
1.14 chs 2138: msr = pmap_interrupts_off();
1.1 matt 2139: for (; va < endva; va += PAGE_SIZE) {
2140: pvo = pmap_pvo_find_va(pm, va, &pteidx);
2141: if (pvo != NULL) {
1.33 chs 2142: pmap_pvo_remove(pvo, pteidx, &pvol);
1.1 matt 2143: }
2144: }
1.14 chs 2145: pmap_interrupts_restore(msr);
1.33 chs 2146: pmap_pvo_free_list(&pvol);
1.48.30.1! bouyer 2147: PMAP_UNLOCK();
1.1 matt 2148: }
2149:
2150: /*
2151: * Get the physical page address for the given pmap/virtual address.
2152: */
1.44 thorpej 2153: bool
1.1 matt 2154: pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap)
2155: {
2156: struct pvo_entry *pvo;
1.2 matt 2157: register_t msr;
1.7 matt 2158:
1.48.30.1! bouyer 2159: PMAP_LOCK();
1.38 sanjayl 2160:
1.7 matt 2161: /*
2162: * If this is a kernel pmap lookup, also check the battable
2163: * and if we get a hit, translate the VA to a PA using the
1.36 nathanw 2164: * BAT entries. Don't check for VM_MAX_KERNEL_ADDRESS is
1.7 matt 2165: * that will wrap back to 0.
2166: */
2167: if (pm == pmap_kernel() &&
2168: (va < VM_MIN_KERNEL_ADDRESS ||
2169: (KERNEL2_SR < 15 && VM_MAX_KERNEL_ADDRESS <= va))) {
1.8 matt 2170: KASSERT((va >> ADDR_SR_SHFT) != USER_SR);
1.38 sanjayl 2171: #if defined (PPC_OEA)
1.24 kleink 2172: if ((MFPVR() >> 16) != MPC601) {
2173: register_t batu = battable[va >> ADDR_SR_SHFT].batu;
2174: if (BAT_VALID_P(batu,0) && BAT_VA_MATCH_P(batu,va)) {
2175: register_t batl =
2176: battable[va >> ADDR_SR_SHFT].batl;
2177: register_t mask =
2178: (~(batu & BAT_BL) << 15) & ~0x1ffffL;
1.29 briggs 2179: if (pap)
2180: *pap = (batl & mask) | (va & ~mask);
1.48.30.1! bouyer 2181: PMAP_UNLOCK();
1.45 thorpej 2182: return true;
1.24 kleink 2183: }
2184: } else {
2185: register_t batu = battable[va >> 23].batu;
2186: register_t batl = battable[va >> 23].batl;
2187: register_t sr = iosrtable[va >> ADDR_SR_SHFT];
2188: if (BAT601_VALID_P(batl) &&
2189: BAT601_VA_MATCH_P(batu, batl, va)) {
2190: register_t mask =
2191: (~(batl & BAT601_BSM) << 17) & ~0x1ffffL;
1.29 briggs 2192: if (pap)
2193: *pap = (batl & mask) | (va & ~mask);
1.48.30.1! bouyer 2194: PMAP_UNLOCK();
1.45 thorpej 2195: return true;
1.24 kleink 2196: } else if (SR601_VALID_P(sr) &&
2197: SR601_PA_MATCH_P(sr, va)) {
1.29 briggs 2198: if (pap)
2199: *pap = va;
1.48.30.1! bouyer 2200: PMAP_UNLOCK();
1.45 thorpej 2201: return true;
1.24 kleink 2202: }
1.7 matt 2203: }
1.48.30.1! bouyer 2204: PMAP_UNLOCK();
1.45 thorpej 2205: return false;
1.38 sanjayl 2206: #elif defined (PPC_OEA64_BRIDGE)
1.48.30.1! bouyer 2207: panic("%s: pm: %s, va: 0x%08lx\n", __func__,
1.38 sanjayl 2208: (pm == pmap_kernel() ? "kernel" : "user"), va);
2209: #elif defined (PPC_OEA64)
2210: #error PPC_OEA64 not supported
2211: #endif /* PPC_OEA */
1.7 matt 2212: }
1.1 matt 2213:
2214: msr = pmap_interrupts_off();
2215: pvo = pmap_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
2216: if (pvo != NULL) {
2217: PMAP_PVO_CHECK(pvo); /* sanity check */
1.29 briggs 2218: if (pap)
2219: *pap = (pvo->pvo_pte.pte_lo & PTE_RPGN)
2220: | (va & ADDR_POFF);
1.1 matt 2221: }
2222: pmap_interrupts_restore(msr);
1.48.30.1! bouyer 2223: PMAP_UNLOCK();
1.1 matt 2224: return pvo != NULL;
2225: }
2226:
2227: /*
2228: * Lower the protection on the specified range of this pmap.
2229: */
2230: void
2231: pmap_protect(pmap_t pm, vaddr_t va, vaddr_t endva, vm_prot_t prot)
2232: {
2233: struct pvo_entry *pvo;
1.2 matt 2234: volatile struct pte *pt;
2235: register_t msr;
1.1 matt 2236: int pteidx;
2237:
2238: /*
2239: * Since this routine only downgrades protection, we should
1.14 chs 2240: * always be called with at least one bit not set.
1.1 matt 2241: */
1.14 chs 2242: KASSERT(prot != VM_PROT_ALL);
1.1 matt 2243:
2244: /*
2245: * If there is no protection, this is equivalent to
2246: * remove the pmap from the pmap.
2247: */
2248: if ((prot & VM_PROT_READ) == 0) {
2249: pmap_remove(pm, va, endva);
2250: return;
2251: }
2252:
1.48.30.1! bouyer 2253: PMAP_LOCK();
! 2254:
1.1 matt 2255: msr = pmap_interrupts_off();
1.6 thorpej 2256: for (; va < endva; va += PAGE_SIZE) {
1.1 matt 2257: pvo = pmap_pvo_find_va(pm, va, &pteidx);
2258: if (pvo == NULL)
2259: continue;
2260: PMAP_PVO_CHECK(pvo); /* sanity check */
2261:
2262: /*
2263: * Revoke executable if asked to do so.
2264: */
2265: if ((prot & VM_PROT_EXECUTE) == 0)
1.14 chs 2266: pvo_clear_exec(pvo);
1.1 matt 2267:
2268: #if 0
2269: /*
2270: * If the page is already read-only, no change
2271: * needs to be made.
2272: */
2273: if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR)
2274: continue;
2275: #endif
2276: /*
2277: * Grab the PTE pointer before we diddle with
2278: * the cached PTE copy.
2279: */
2280: pt = pmap_pvo_to_pte(pvo, pteidx);
2281: /*
2282: * Change the protection of the page.
2283: */
2284: pvo->pvo_pte.pte_lo &= ~PTE_PP;
2285: pvo->pvo_pte.pte_lo |= PTE_BR;
2286:
2287: /*
2288: * If the PVO is in the page table, update
2289: * that pte at well.
2290: */
2291: if (pt != NULL) {
2292: pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
1.12 matt 2293: PVO_WHERE(pvo, PMAP_PROTECT);
1.1 matt 2294: PMAPCOUNT(ptes_changed);
2295: }
2296:
2297: PMAP_PVO_CHECK(pvo); /* sanity check */
2298: }
2299: pmap_interrupts_restore(msr);
1.48.30.1! bouyer 2300: PMAP_UNLOCK();
1.1 matt 2301: }
2302:
2303: void
2304: pmap_unwire(pmap_t pm, vaddr_t va)
2305: {
2306: struct pvo_entry *pvo;
1.2 matt 2307: register_t msr;
1.1 matt 2308:
1.48.30.1! bouyer 2309: PMAP_LOCK();
1.1 matt 2310: msr = pmap_interrupts_off();
2311: pvo = pmap_pvo_find_va(pm, va, NULL);
2312: if (pvo != NULL) {
1.39 matt 2313: if (PVO_WIRED_P(pvo)) {
1.1 matt 2314: pvo->pvo_vaddr &= ~PVO_WIRED;
2315: pm->pm_stats.wired_count--;
2316: }
2317: PMAP_PVO_CHECK(pvo); /* sanity check */
2318: }
2319: pmap_interrupts_restore(msr);
1.48.30.1! bouyer 2320: PMAP_UNLOCK();
1.1 matt 2321: }
2322:
2323: /*
2324: * Lower the protection on the specified physical page.
2325: */
2326: void
2327: pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
2328: {
1.33 chs 2329: struct pvo_head *pvo_head, pvol;
1.1 matt 2330: struct pvo_entry *pvo, *next_pvo;
1.2 matt 2331: volatile struct pte *pt;
2332: register_t msr;
1.1 matt 2333:
1.48.30.1! bouyer 2334: PMAP_LOCK();
! 2335:
1.14 chs 2336: KASSERT(prot != VM_PROT_ALL);
1.33 chs 2337: LIST_INIT(&pvol);
1.1 matt 2338: msr = pmap_interrupts_off();
2339:
2340: /*
2341: * When UVM reuses a page, it does a pmap_page_protect with
2342: * VM_PROT_NONE. At that point, we can clear the exec flag
2343: * since we know the page will have different contents.
2344: */
2345: if ((prot & VM_PROT_READ) == 0) {
2346: DPRINTFN(EXEC, ("[pmap_page_protect: %#lx: clear-exec]\n",
1.34 yamt 2347: VM_PAGE_TO_PHYS(pg)));
1.1 matt 2348: if (pmap_attr_fetch(pg) & PTE_EXEC) {
2349: PMAPCOUNT(exec_uncached_page_protect);
2350: pmap_attr_clear(pg, PTE_EXEC);
2351: }
2352: }
2353:
2354: pvo_head = vm_page_to_pvoh(pg);
2355: for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) {
2356: next_pvo = LIST_NEXT(pvo, pvo_vlink);
2357: PMAP_PVO_CHECK(pvo); /* sanity check */
2358:
2359: /*
2360: * Downgrading to no mapping at all, we just remove the entry.
2361: */
2362: if ((prot & VM_PROT_READ) == 0) {
1.33 chs 2363: pmap_pvo_remove(pvo, -1, &pvol);
1.1 matt 2364: continue;
2365: }
2366:
2367: /*
2368: * If EXEC permission is being revoked, just clear the
2369: * flag in the PVO.
2370: */
2371: if ((prot & VM_PROT_EXECUTE) == 0)
1.14 chs 2372: pvo_clear_exec(pvo);
1.1 matt 2373:
2374: /*
2375: * If this entry is already RO, don't diddle with the
2376: * page table.
2377: */
2378: if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) {
2379: PMAP_PVO_CHECK(pvo);
2380: continue;
2381: }
2382:
2383: /*
2384: * Grab the PTE before the we diddle the bits so
2385: * pvo_to_pte can verify the pte contents are as
2386: * expected.
2387: */
2388: pt = pmap_pvo_to_pte(pvo, -1);
2389: pvo->pvo_pte.pte_lo &= ~PTE_PP;
2390: pvo->pvo_pte.pte_lo |= PTE_BR;
2391: if (pt != NULL) {
2392: pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
1.12 matt 2393: PVO_WHERE(pvo, PMAP_PAGE_PROTECT);
1.1 matt 2394: PMAPCOUNT(ptes_changed);
2395: }
2396: PMAP_PVO_CHECK(pvo); /* sanity check */
2397: }
2398: pmap_interrupts_restore(msr);
1.33 chs 2399: pmap_pvo_free_list(&pvol);
1.48.30.1! bouyer 2400:
! 2401: PMAP_UNLOCK();
1.1 matt 2402: }
2403:
2404: /*
2405: * Activate the address space for the specified process. If the process
2406: * is the current process, load the new MMU context.
2407: */
2408: void
2409: pmap_activate(struct lwp *l)
2410: {
2411: struct pcb *pcb = &l->l_addr->u_pcb;
2412: pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap;
2413:
2414: DPRINTFN(ACTIVATE,
2415: ("pmap_activate: lwp %p (curlwp %p)\n", l, curlwp));
2416:
2417: /*
2418: * XXX Normally performed in cpu_fork().
2419: */
1.13 matt 2420: pcb->pcb_pm = pmap;
1.17 matt 2421:
2422: /*
2423: * In theory, the SR registers need only be valid on return
2424: * to user space wait to do them there.
2425: */
2426: if (l == curlwp) {
2427: /* Store pointer to new current pmap. */
2428: curpm = pmap;
2429: }
1.1 matt 2430: }
2431:
2432: /*
2433: * Deactivate the specified process's address space.
2434: */
2435: void
2436: pmap_deactivate(struct lwp *l)
2437: {
2438: }
2439:
1.44 thorpej 2440: bool
1.1 matt 2441: pmap_query_bit(struct vm_page *pg, int ptebit)
2442: {
2443: struct pvo_entry *pvo;
1.2 matt 2444: volatile struct pte *pt;
2445: register_t msr;
1.1 matt 2446:
1.48.30.1! bouyer 2447: PMAP_LOCK();
! 2448:
! 2449: if (pmap_attr_fetch(pg) & ptebit) {
! 2450: PMAP_UNLOCK();
1.45 thorpej 2451: return true;
1.48.30.1! bouyer 2452: }
1.14 chs 2453:
1.1 matt 2454: msr = pmap_interrupts_off();
2455: LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) {
2456: PMAP_PVO_CHECK(pvo); /* sanity check */
2457: /*
2458: * See if we saved the bit off. If so cache, it and return
2459: * success.
2460: */
2461: if (pvo->pvo_pte.pte_lo & ptebit) {
2462: pmap_attr_save(pg, ptebit);
2463: PMAP_PVO_CHECK(pvo); /* sanity check */
2464: pmap_interrupts_restore(msr);
1.48.30.1! bouyer 2465: PMAP_UNLOCK();
1.45 thorpej 2466: return true;
1.1 matt 2467: }
2468: }
2469: /*
2470: * No luck, now go thru the hard part of looking at the ptes
2471: * themselves. Sync so any pending REF/CHG bits are flushed
2472: * to the PTEs.
2473: */
2474: SYNC();
2475: LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) {
2476: PMAP_PVO_CHECK(pvo); /* sanity check */
2477: /*
2478: * See if this pvo have a valid PTE. If so, fetch the
2479: * REF/CHG bits from the valid PTE. If the appropriate
2480: * ptebit is set, cache, it and return success.
2481: */
2482: pt = pmap_pvo_to_pte(pvo, -1);
2483: if (pt != NULL) {
2484: pmap_pte_synch(pt, &pvo->pvo_pte);
2485: if (pvo->pvo_pte.pte_lo & ptebit) {
2486: pmap_attr_save(pg, ptebit);
2487: PMAP_PVO_CHECK(pvo); /* sanity check */
2488: pmap_interrupts_restore(msr);
1.48.30.1! bouyer 2489: PMAP_UNLOCK();
1.45 thorpej 2490: return true;
1.1 matt 2491: }
2492: }
2493: }
2494: pmap_interrupts_restore(msr);
1.48.30.1! bouyer 2495: PMAP_UNLOCK();
1.45 thorpej 2496: return false;
1.1 matt 2497: }
2498:
1.44 thorpej 2499: bool
1.1 matt 2500: pmap_clear_bit(struct vm_page *pg, int ptebit)
2501: {
2502: struct pvo_head *pvoh = vm_page_to_pvoh(pg);
2503: struct pvo_entry *pvo;
1.2 matt 2504: volatile struct pte *pt;
2505: register_t msr;
1.1 matt 2506: int rv = 0;
2507:
1.48.30.1! bouyer 2508: PMAP_LOCK();
1.1 matt 2509: msr = pmap_interrupts_off();
2510:
2511: /*
2512: * Fetch the cache value
2513: */
2514: rv |= pmap_attr_fetch(pg);
2515:
2516: /*
2517: * Clear the cached value.
2518: */
2519: pmap_attr_clear(pg, ptebit);
2520:
2521: /*
2522: * Sync so any pending REF/CHG bits are flushed to the PTEs (so we
2523: * can reset the right ones). Note that since the pvo entries and
2524: * list heads are accessed via BAT0 and are never placed in the
2525: * page table, we don't have to worry about further accesses setting
2526: * the REF/CHG bits.
2527: */
2528: SYNC();
2529:
2530: /*
2531: * For each pvo entry, clear pvo's ptebit. If this pvo have a
2532: * valid PTE. If so, clear the ptebit from the valid PTE.
2533: */
2534: LIST_FOREACH(pvo, pvoh, pvo_vlink) {
2535: PMAP_PVO_CHECK(pvo); /* sanity check */
2536: pt = pmap_pvo_to_pte(pvo, -1);
2537: if (pt != NULL) {
2538: /*
2539: * Only sync the PTE if the bit we are looking
2540: * for is not already set.
2541: */
2542: if ((pvo->pvo_pte.pte_lo & ptebit) == 0)
2543: pmap_pte_synch(pt, &pvo->pvo_pte);
2544: /*
2545: * If the bit we are looking for was already set,
2546: * clear that bit in the pte.
2547: */
2548: if (pvo->pvo_pte.pte_lo & ptebit)
2549: pmap_pte_clear(pt, PVO_VADDR(pvo), ptebit);
2550: }
2551: rv |= pvo->pvo_pte.pte_lo & (PTE_CHG|PTE_REF);
2552: pvo->pvo_pte.pte_lo &= ~ptebit;
2553: PMAP_PVO_CHECK(pvo); /* sanity check */
2554: }
2555: pmap_interrupts_restore(msr);
1.14 chs 2556:
1.1 matt 2557: /*
2558: * If we are clearing the modify bit and this page was marked EXEC
2559: * and the user of the page thinks the page was modified, then we
2560: * need to clean it from the icache if it's mapped or clear the EXEC
2561: * bit if it's not mapped. The page itself might not have the CHG
2562: * bit set if the modification was done via DMA to the page.
2563: */
2564: if ((ptebit & PTE_CHG) && (rv & PTE_EXEC)) {
2565: if (LIST_EMPTY(pvoh)) {
2566: DPRINTFN(EXEC, ("[pmap_clear_bit: %#lx: clear-exec]\n",
1.34 yamt 2567: VM_PAGE_TO_PHYS(pg)));
1.1 matt 2568: pmap_attr_clear(pg, PTE_EXEC);
2569: PMAPCOUNT(exec_uncached_clear_modify);
2570: } else {
2571: DPRINTFN(EXEC, ("[pmap_clear_bit: %#lx: syncicache]\n",
1.34 yamt 2572: VM_PAGE_TO_PHYS(pg)));
2573: pmap_syncicache(VM_PAGE_TO_PHYS(pg), PAGE_SIZE);
1.1 matt 2574: PMAPCOUNT(exec_synced_clear_modify);
2575: }
2576: }
1.48.30.1! bouyer 2577: PMAP_UNLOCK();
1.1 matt 2578: return (rv & ptebit) != 0;
2579: }
2580:
2581: void
2582: pmap_procwr(struct proc *p, vaddr_t va, size_t len)
2583: {
2584: struct pvo_entry *pvo;
2585: size_t offset = va & ADDR_POFF;
2586: int s;
2587:
1.48.30.1! bouyer 2588: PMAP_LOCK();
1.1 matt 2589: s = splvm();
2590: while (len > 0) {
1.6 thorpej 2591: size_t seglen = PAGE_SIZE - offset;
1.1 matt 2592: if (seglen > len)
2593: seglen = len;
2594: pvo = pmap_pvo_find_va(p->p_vmspace->vm_map.pmap, va, NULL);
1.39 matt 2595: if (pvo != NULL && PVO_EXECUTABLE_P(pvo)) {
1.1 matt 2596: pmap_syncicache(
2597: (pvo->pvo_pte.pte_lo & PTE_RPGN) | offset, seglen);
2598: PMAP_PVO_CHECK(pvo);
2599: }
2600: va += seglen;
2601: len -= seglen;
2602: offset = 0;
2603: }
2604: splx(s);
1.48.30.1! bouyer 2605: PMAP_UNLOCK();
1.1 matt 2606: }
2607:
2608: #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
2609: void
1.2 matt 2610: pmap_pte_print(volatile struct pte *pt)
1.1 matt 2611: {
2612: printf("PTE %p: ", pt);
1.38 sanjayl 2613:
2614: #if defined(PPC_OEA)
1.1 matt 2615: /* High word: */
1.2 matt 2616: printf("0x%08lx: [", pt->pte_hi);
1.38 sanjayl 2617: #elif defined (PPC_OEA64_BRIDGE)
2618: printf("0x%016llx: [", pt->pte_hi);
2619: #else /* PPC_OEA64 */
2620: printf("0x%016lx: [", pt->pte_hi);
2621: #endif /* PPC_OEA */
2622:
1.1 matt 2623: printf("%c ", (pt->pte_hi & PTE_VALID) ? 'v' : 'i');
2624: printf("%c ", (pt->pte_hi & PTE_HID) ? 'h' : '-');
1.38 sanjayl 2625:
2626: #if defined (PPC_OEA)
1.2 matt 2627: printf("0x%06lx 0x%02lx",
1.1 matt 2628: (pt->pte_hi &~ PTE_VALID)>>PTE_VSID_SHFT,
2629: pt->pte_hi & PTE_API);
2630: printf(" (va 0x%08lx)] ", pmap_pte_to_va(pt));
1.38 sanjayl 2631: #elif defined (PPC_OEA64)
2632: printf("0x%06lx 0x%02lx",
2633: (pt->pte_hi &~ PTE_VALID)>>PTE_VSID_SHFT,
2634: pt->pte_hi & PTE_API);
2635: printf(" (va 0x%016lx)] ", pmap_pte_to_va(pt));
2636: #else
2637: /* PPC_OEA64_BRIDGE */
2638: printf("0x%06llx 0x%02llx",
2639: (pt->pte_hi &~ PTE_VALID)>>PTE_VSID_SHFT,
2640: pt->pte_hi & PTE_API);
2641: printf(" (va 0x%08lx)] ", pmap_pte_to_va(pt));
2642: #endif /* PPC_OEA */
2643:
1.1 matt 2644: /* Low word: */
1.38 sanjayl 2645: #if defined (PPC_OEA)
1.2 matt 2646: printf(" 0x%08lx: [", pt->pte_lo);
2647: printf("0x%05lx... ", pt->pte_lo >> 12);
1.38 sanjayl 2648: #elif defined (PPC_OEA64)
2649: printf(" 0x%016lx: [", pt->pte_lo);
2650: printf("0x%012lx... ", pt->pte_lo >> 12);
2651: #else /* PPC_OEA64_BRIDGE */
2652: printf(" 0x%016llx: [", pt->pte_lo);
2653: printf("0x%012llx... ", pt->pte_lo >> 12);
2654: #endif
1.1 matt 2655: printf("%c ", (pt->pte_lo & PTE_REF) ? 'r' : 'u');
2656: printf("%c ", (pt->pte_lo & PTE_CHG) ? 'c' : 'n');
2657: printf("%c", (pt->pte_lo & PTE_W) ? 'w' : '.');
2658: printf("%c", (pt->pte_lo & PTE_I) ? 'i' : '.');
2659: printf("%c", (pt->pte_lo & PTE_M) ? 'm' : '.');
2660: printf("%c ", (pt->pte_lo & PTE_G) ? 'g' : '.');
2661: switch (pt->pte_lo & PTE_PP) {
2662: case PTE_BR: printf("br]\n"); break;
2663: case PTE_BW: printf("bw]\n"); break;
2664: case PTE_SO: printf("so]\n"); break;
2665: case PTE_SW: printf("sw]\n"); break;
2666: }
2667: }
2668: #endif
2669:
2670: #if defined(DDB)
2671: void
2672: pmap_pteg_check(void)
2673: {
1.2 matt 2674: volatile struct pte *pt;
1.1 matt 2675: int i;
2676: int ptegidx;
2677: u_int p_valid = 0;
2678: u_int s_valid = 0;
2679: u_int invalid = 0;
1.38 sanjayl 2680:
1.1 matt 2681: for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) {
2682: for (pt = pmap_pteg_table[ptegidx].pt, i = 8; --i >= 0; pt++) {
2683: if (pt->pte_hi & PTE_VALID) {
2684: if (pt->pte_hi & PTE_HID)
2685: s_valid++;
2686: else
1.38 sanjayl 2687: {
1.1 matt 2688: p_valid++;
1.38 sanjayl 2689: }
1.1 matt 2690: } else
2691: invalid++;
2692: }
2693: }
2694: printf("pteg_check: v(p) %#x (%d), v(s) %#x (%d), i %#x (%d)\n",
2695: p_valid, p_valid, s_valid, s_valid,
2696: invalid, invalid);
2697: }
2698:
2699: void
2700: pmap_print_mmuregs(void)
2701: {
2702: int i;
2703: u_int cpuvers;
1.18 matt 2704: #ifndef PPC_OEA64
1.1 matt 2705: vaddr_t addr;
1.2 matt 2706: register_t soft_sr[16];
1.18 matt 2707: #endif
1.38 sanjayl 2708: #if defined (PPC_OEA) && !defined (PPC_OEA64) && !defined (PPC_OEA64_BRIDGE)
1.1 matt 2709: struct bat soft_ibat[4];
2710: struct bat soft_dbat[4];
1.38 sanjayl 2711: #endif
1.2 matt 2712: register_t sdr1;
1.1 matt 2713:
2714: cpuvers = MFPVR() >> 16;
1.35 perry 2715: __asm volatile ("mfsdr1 %0" : "=r"(sdr1));
1.18 matt 2716: #ifndef PPC_OEA64
1.16 kleink 2717: addr = 0;
1.27 chs 2718: for (i = 0; i < 16; i++) {
1.1 matt 2719: soft_sr[i] = MFSRIN(addr);
2720: addr += (1 << ADDR_SR_SHFT);
2721: }
1.18 matt 2722: #endif
1.1 matt 2723:
1.38 sanjayl 2724: #if defined(PPC_OEA) && !defined (PPC_OEA64) && !defined (PPC_OEA64_BRIDGE)
1.1 matt 2725: /* read iBAT (601: uBAT) registers */
1.35 perry 2726: __asm volatile ("mfibatu %0,0" : "=r"(soft_ibat[0].batu));
2727: __asm volatile ("mfibatl %0,0" : "=r"(soft_ibat[0].batl));
2728: __asm volatile ("mfibatu %0,1" : "=r"(soft_ibat[1].batu));
2729: __asm volatile ("mfibatl %0,1" : "=r"(soft_ibat[1].batl));
2730: __asm volatile ("mfibatu %0,2" : "=r"(soft_ibat[2].batu));
2731: __asm volatile ("mfibatl %0,2" : "=r"(soft_ibat[2].batl));
2732: __asm volatile ("mfibatu %0,3" : "=r"(soft_ibat[3].batu));
2733: __asm volatile ("mfibatl %0,3" : "=r"(soft_ibat[3].batl));
1.1 matt 2734:
2735:
2736: if (cpuvers != MPC601) {
2737: /* read dBAT registers */
1.35 perry 2738: __asm volatile ("mfdbatu %0,0" : "=r"(soft_dbat[0].batu));
2739: __asm volatile ("mfdbatl %0,0" : "=r"(soft_dbat[0].batl));
2740: __asm volatile ("mfdbatu %0,1" : "=r"(soft_dbat[1].batu));
2741: __asm volatile ("mfdbatl %0,1" : "=r"(soft_dbat[1].batl));
2742: __asm volatile ("mfdbatu %0,2" : "=r"(soft_dbat[2].batu));
2743: __asm volatile ("mfdbatl %0,2" : "=r"(soft_dbat[2].batl));
2744: __asm volatile ("mfdbatu %0,3" : "=r"(soft_dbat[3].batu));
2745: __asm volatile ("mfdbatl %0,3" : "=r"(soft_dbat[3].batl));
1.1 matt 2746: }
1.38 sanjayl 2747: #endif
1.1 matt 2748:
1.18 matt 2749: printf("SDR1:\t0x%lx\n", (long) sdr1);
2750: #ifndef PPC_OEA64
1.1 matt 2751: printf("SR[]:\t");
1.27 chs 2752: for (i = 0; i < 4; i++)
1.38 sanjayl 2753: printf("0x%08lx, ", (long) soft_sr[i]);
1.1 matt 2754: printf("\n\t");
1.27 chs 2755: for ( ; i < 8; i++)
1.38 sanjayl 2756: printf("0x%08lx, ", (long) soft_sr[i]);
1.1 matt 2757: printf("\n\t");
1.27 chs 2758: for ( ; i < 12; i++)
1.38 sanjayl 2759: printf("0x%08lx, ", (long) soft_sr[i]);
1.1 matt 2760: printf("\n\t");
1.27 chs 2761: for ( ; i < 16; i++)
1.38 sanjayl 2762: printf("0x%08lx, ", (long) soft_sr[i]);
1.1 matt 2763: printf("\n");
1.18 matt 2764: #endif
1.1 matt 2765:
1.38 sanjayl 2766: #if defined(PPC_OEA) && !defined (PPC_OEA64) && !defined (PPC_OEA64_BRIDGE)
1.1 matt 2767: printf("%cBAT[]:\t", cpuvers == MPC601 ? 'u' : 'i');
1.27 chs 2768: for (i = 0; i < 4; i++) {
1.2 matt 2769: printf("0x%08lx 0x%08lx, ",
1.1 matt 2770: soft_ibat[i].batu, soft_ibat[i].batl);
2771: if (i == 1)
2772: printf("\n\t");
2773: }
2774: if (cpuvers != MPC601) {
2775: printf("\ndBAT[]:\t");
1.27 chs 2776: for (i = 0; i < 4; i++) {
1.2 matt 2777: printf("0x%08lx 0x%08lx, ",
1.1 matt 2778: soft_dbat[i].batu, soft_dbat[i].batl);
2779: if (i == 1)
2780: printf("\n\t");
2781: }
2782: }
2783: printf("\n");
1.38 sanjayl 2784: #endif /* PPC_OEA... */
1.1 matt 2785: }
2786:
2787: void
2788: pmap_print_pte(pmap_t pm, vaddr_t va)
2789: {
2790: struct pvo_entry *pvo;
1.2 matt 2791: volatile struct pte *pt;
1.1 matt 2792: int pteidx;
2793:
2794: pvo = pmap_pvo_find_va(pm, va, &pteidx);
2795: if (pvo != NULL) {
2796: pt = pmap_pvo_to_pte(pvo, pteidx);
2797: if (pt != NULL) {
1.38 sanjayl 2798: #if defined (PPC_OEA) || defined (PPC_OEA64)
1.2 matt 2799: printf("VA %#lx -> %p -> %s %#lx, %#lx\n",
1.1 matt 2800: va, pt,
2801: pt->pte_hi & PTE_HID ? "(sec)" : "(pri)",
2802: pt->pte_hi, pt->pte_lo);
1.38 sanjayl 2803: #else /* PPC_OEA64_BRIDGE */
2804: printf("VA %#lx -> %p -> %s %#llx, %#llx\n",
2805: va, pt,
2806: pt->pte_hi & PTE_HID ? "(sec)" : "(pri)",
2807: pt->pte_hi, pt->pte_lo);
2808: #endif
1.1 matt 2809: } else {
2810: printf("No valid PTE found\n");
2811: }
2812: } else {
2813: printf("Address not in pmap\n");
2814: }
2815: }
2816:
2817: void
2818: pmap_pteg_dist(void)
2819: {
2820: struct pvo_entry *pvo;
2821: int ptegidx;
2822: int depth;
2823: int max_depth = 0;
2824: unsigned int depths[64];
2825:
2826: memset(depths, 0, sizeof(depths));
2827: for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) {
2828: depth = 0;
2829: TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
2830: depth++;
2831: }
2832: if (depth > max_depth)
2833: max_depth = depth;
2834: if (depth > 63)
2835: depth = 63;
2836: depths[depth]++;
2837: }
2838:
2839: for (depth = 0; depth < 64; depth++) {
2840: printf(" [%2d]: %8u", depth, depths[depth]);
2841: if ((depth & 3) == 3)
2842: printf("\n");
2843: if (depth == max_depth)
2844: break;
2845: }
2846: if ((depth & 3) != 3)
2847: printf("\n");
2848: printf("Max depth found was %d\n", max_depth);
2849: }
2850: #endif /* DEBUG */
2851:
2852: #if defined(PMAPCHECK) || defined(DEBUG)
2853: void
2854: pmap_pvo_verify(void)
2855: {
2856: int ptegidx;
2857: int s;
2858:
2859: s = splvm();
2860: for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) {
2861: struct pvo_entry *pvo;
2862: TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
2863: if ((uintptr_t) pvo >= SEGMENT_LENGTH)
2864: panic("pmap_pvo_verify: invalid pvo %p "
2865: "on list %#x", pvo, ptegidx);
2866: pmap_pvo_check(pvo);
2867: }
2868: }
2869: splx(s);
2870: }
2871: #endif /* PMAPCHECK */
2872:
2873:
2874: void *
2875: pmap_pool_ualloc(struct pool *pp, int flags)
2876: {
2877: struct pvo_page *pvop;
2878:
1.48.30.1! bouyer 2879: if (uvm.page_init_done != true) {
! 2880: return (void *) uvm_pageboot_alloc(PAGE_SIZE);
! 2881: }
! 2882:
! 2883: PMAP_LOCK();
1.1 matt 2884: pvop = SIMPLEQ_FIRST(&pmap_upvop_head);
2885: if (pvop != NULL) {
2886: pmap_upvop_free--;
2887: SIMPLEQ_REMOVE_HEAD(&pmap_upvop_head, pvop_link);
1.48.30.1! bouyer 2888: PMAP_UNLOCK();
1.1 matt 2889: return pvop;
2890: }
1.48.30.1! bouyer 2891: PMAP_UNLOCK();
1.1 matt 2892: return pmap_pool_malloc(pp, flags);
2893: }
2894:
2895: void *
2896: pmap_pool_malloc(struct pool *pp, int flags)
2897: {
2898: struct pvo_page *pvop;
2899: struct vm_page *pg;
2900:
1.48.30.1! bouyer 2901: PMAP_LOCK();
1.1 matt 2902: pvop = SIMPLEQ_FIRST(&pmap_mpvop_head);
2903: if (pvop != NULL) {
2904: pmap_mpvop_free--;
2905: SIMPLEQ_REMOVE_HEAD(&pmap_mpvop_head, pvop_link);
1.48.30.1! bouyer 2906: PMAP_UNLOCK();
1.1 matt 2907: return pvop;
2908: }
1.48.30.1! bouyer 2909: PMAP_UNLOCK();
1.1 matt 2910: again:
2911: pg = uvm_pagealloc_strat(NULL, 0, NULL, UVM_PGA_USERESERVE,
2912: UVM_PGA_STRAT_ONLY, VM_FREELIST_FIRST256);
2913: if (__predict_false(pg == NULL)) {
2914: if (flags & PR_WAITOK) {
2915: uvm_wait("plpg");
2916: goto again;
2917: } else {
2918: return (0);
2919: }
2920: }
2921: return (void *) VM_PAGE_TO_PHYS(pg);
2922: }
2923:
2924: void
2925: pmap_pool_ufree(struct pool *pp, void *va)
2926: {
2927: struct pvo_page *pvop;
2928: #if 0
2929: if (PHYS_TO_VM_PAGE((paddr_t) va) != NULL) {
2930: pmap_pool_mfree(va, size, tag);
2931: return;
2932: }
2933: #endif
1.48.30.1! bouyer 2934: PMAP_LOCK();
1.1 matt 2935: pvop = va;
2936: SIMPLEQ_INSERT_HEAD(&pmap_upvop_head, pvop, pvop_link);
2937: pmap_upvop_free++;
2938: if (pmap_upvop_free > pmap_upvop_maxfree)
2939: pmap_upvop_maxfree = pmap_upvop_free;
1.48.30.1! bouyer 2940: PMAP_UNLOCK();
1.1 matt 2941: }
2942:
2943: void
2944: pmap_pool_mfree(struct pool *pp, void *va)
2945: {
2946: struct pvo_page *pvop;
2947:
1.48.30.1! bouyer 2948: PMAP_LOCK();
1.1 matt 2949: pvop = va;
2950: SIMPLEQ_INSERT_HEAD(&pmap_mpvop_head, pvop, pvop_link);
2951: pmap_mpvop_free++;
2952: if (pmap_mpvop_free > pmap_mpvop_maxfree)
2953: pmap_mpvop_maxfree = pmap_mpvop_free;
1.48.30.1! bouyer 2954: PMAP_UNLOCK();
1.1 matt 2955: #if 0
2956: uvm_pagefree(PHYS_TO_VM_PAGE((paddr_t) va));
2957: #endif
2958: }
2959:
2960: /*
2961: * This routine in bootstraping to steal to-be-managed memory (which will
2962: * then be unmanaged). We use it to grab from the first 256MB for our
2963: * pmap needs and above 256MB for other stuff.
2964: */
2965: vaddr_t
1.10 thorpej 2966: pmap_steal_memory(vsize_t vsize, vaddr_t *vstartp, vaddr_t *vendp)
1.1 matt 2967: {
2968: vsize_t size;
2969: vaddr_t va;
2970: paddr_t pa = 0;
2971: int npgs, bank;
2972: struct vm_physseg *ps;
2973:
1.45 thorpej 2974: if (uvm.page_init_done == true)
1.1 matt 2975: panic("pmap_steal_memory: called _after_ bootstrap");
2976:
1.10 thorpej 2977: *vstartp = VM_MIN_KERNEL_ADDRESS;
2978: *vendp = VM_MAX_KERNEL_ADDRESS;
2979:
1.1 matt 2980: size = round_page(vsize);
2981: npgs = atop(size);
2982:
2983: /*
2984: * PA 0 will never be among those given to UVM so we can use it
2985: * to indicate we couldn't steal any memory.
2986: */
2987: for (ps = vm_physmem, bank = 0; bank < vm_nphysseg; bank++, ps++) {
2988: if (ps->free_list == VM_FREELIST_FIRST256 &&
2989: ps->avail_end - ps->avail_start >= npgs) {
2990: pa = ptoa(ps->avail_start);
2991: break;
2992: }
2993: }
2994:
2995: if (pa == 0)
2996: panic("pmap_steal_memory: no approriate memory to steal!");
2997:
2998: ps->avail_start += npgs;
2999: ps->start += npgs;
3000:
3001: /*
3002: * If we've used up all the pages in the segment, remove it and
3003: * compact the list.
3004: */
3005: if (ps->avail_start == ps->end) {
3006: /*
3007: * If this was the last one, then a very bad thing has occurred
3008: */
3009: if (--vm_nphysseg == 0)
3010: panic("pmap_steal_memory: out of memory!");
3011:
3012: printf("pmap_steal_memory: consumed bank %d\n", bank);
3013: for (; bank < vm_nphysseg; bank++, ps++) {
3014: ps[0] = ps[1];
3015: }
3016: }
3017:
3018: va = (vaddr_t) pa;
1.46 christos 3019: memset((void *) va, 0, size);
1.1 matt 3020: pmap_pages_stolen += npgs;
3021: #ifdef DEBUG
3022: if (pmapdebug && npgs > 1) {
3023: u_int cnt = 0;
3024: for (bank = 0, ps = vm_physmem; bank < vm_nphysseg; bank++, ps++)
3025: cnt += ps->avail_end - ps->avail_start;
3026: printf("pmap_steal_memory: stole %u (total %u) pages (%u left)\n",
3027: npgs, pmap_pages_stolen, cnt);
3028: }
3029: #endif
3030:
3031: return va;
3032: }
3033:
3034: /*
3035: * Find a chuck of memory with right size and alignment.
3036: */
3037: void *
3038: pmap_boot_find_memory(psize_t size, psize_t alignment, int at_end)
3039: {
3040: struct mem_region *mp;
3041: paddr_t s, e;
3042: int i, j;
3043:
3044: size = round_page(size);
3045:
3046: DPRINTFN(BOOT,
3047: ("pmap_boot_find_memory: size=%lx, alignment=%lx, at_end=%d",
3048: size, alignment, at_end));
3049:
1.6 thorpej 3050: if (alignment < PAGE_SIZE || (alignment & (alignment-1)) != 0)
1.1 matt 3051: panic("pmap_boot_find_memory: invalid alignment %lx",
3052: alignment);
3053:
3054: if (at_end) {
1.6 thorpej 3055: if (alignment != PAGE_SIZE)
1.1 matt 3056: panic("pmap_boot_find_memory: invalid ending "
3057: "alignment %lx", alignment);
3058:
3059: for (mp = &avail[avail_cnt-1]; mp >= avail; mp--) {
3060: s = mp->start + mp->size - size;
3061: if (s >= mp->start && mp->size >= size) {
3062: DPRINTFN(BOOT,(": %lx\n", s));
3063: DPRINTFN(BOOT,
3064: ("pmap_boot_find_memory: b-avail[%d] start "
3065: "0x%lx size 0x%lx\n", mp - avail,
3066: mp->start, mp->size));
3067: mp->size -= size;
3068: DPRINTFN(BOOT,
3069: ("pmap_boot_find_memory: a-avail[%d] start "
3070: "0x%lx size 0x%lx\n", mp - avail,
3071: mp->start, mp->size));
3072: return (void *) s;
3073: }
3074: }
3075: panic("pmap_boot_find_memory: no available memory");
3076: }
3077:
3078: for (mp = avail, i = 0; i < avail_cnt; i++, mp++) {
3079: s = (mp->start + alignment - 1) & ~(alignment-1);
3080: e = s + size;
3081:
3082: /*
3083: * Is the calculated region entirely within the region?
3084: */
3085: if (s < mp->start || e > mp->start + mp->size)
3086: continue;
3087:
3088: DPRINTFN(BOOT,(": %lx\n", s));
3089: if (s == mp->start) {
3090: /*
3091: * If the block starts at the beginning of region,
3092: * adjust the size & start. (the region may now be
3093: * zero in length)
3094: */
3095: DPRINTFN(BOOT,
3096: ("pmap_boot_find_memory: b-avail[%d] start "
3097: "0x%lx size 0x%lx\n", i, mp->start, mp->size));
3098: mp->start += size;
3099: mp->size -= size;
3100: DPRINTFN(BOOT,
3101: ("pmap_boot_find_memory: a-avail[%d] start "
3102: "0x%lx size 0x%lx\n", i, mp->start, mp->size));
3103: } else if (e == mp->start + mp->size) {
3104: /*
3105: * If the block starts at the beginning of region,
3106: * adjust only the size.
3107: */
3108: DPRINTFN(BOOT,
3109: ("pmap_boot_find_memory: b-avail[%d] start "
3110: "0x%lx size 0x%lx\n", i, mp->start, mp->size));
3111: mp->size -= size;
3112: DPRINTFN(BOOT,
3113: ("pmap_boot_find_memory: a-avail[%d] start "
3114: "0x%lx size 0x%lx\n", i, mp->start, mp->size));
3115: } else {
3116: /*
3117: * Block is in the middle of the region, so we
3118: * have to split it in two.
3119: */
3120: for (j = avail_cnt; j > i + 1; j--) {
3121: avail[j] = avail[j-1];
3122: }
3123: DPRINTFN(BOOT,
3124: ("pmap_boot_find_memory: b-avail[%d] start "
3125: "0x%lx size 0x%lx\n", i, mp->start, mp->size));
3126: mp[1].start = e;
3127: mp[1].size = mp[0].start + mp[0].size - e;
3128: mp[0].size = s - mp[0].start;
3129: avail_cnt++;
3130: for (; i < avail_cnt; i++) {
3131: DPRINTFN(BOOT,
3132: ("pmap_boot_find_memory: a-avail[%d] "
3133: "start 0x%lx size 0x%lx\n", i,
3134: avail[i].start, avail[i].size));
3135: }
3136: }
3137: return (void *) s;
3138: }
3139: panic("pmap_boot_find_memory: not enough memory for "
3140: "%lx/%lx allocation?", size, alignment);
3141: }
3142:
1.38 sanjayl 3143: /* XXXSL: we dont have any BATs to do this, map in Segment 0 1:1 using page tables */
3144: #if defined (PPC_OEA64_BRIDGE)
3145: int
3146: pmap_setup_segment0_map(int use_large_pages, ...)
3147: {
3148: vaddr_t va;
3149:
3150: register_t pte_lo = 0x0;
3151: int ptegidx = 0, i = 0;
3152: struct pte pte;
3153: va_list ap;
3154:
3155: /* Coherent + Supervisor RW, no user access */
3156: pte_lo = PTE_M;
3157:
3158: /* XXXSL
3159: * Map in 1st segment 1:1, we'll be careful not to spill kernel entries later,
3160: * these have to take priority.
3161: */
3162: for (va = 0x0; va < SEGMENT_LENGTH; va += 0x1000) {
3163: ptegidx = va_to_pteg(pmap_kernel(), va);
3164: pmap_pte_create(&pte, pmap_kernel(), va, va | pte_lo);
3165: i = pmap_pte_insert(ptegidx, &pte);
3166: }
3167:
3168: va_start(ap, use_large_pages);
3169: while (1) {
3170: paddr_t pa;
3171: size_t size;
3172:
3173: va = va_arg(ap, vaddr_t);
3174:
3175: if (va == 0)
3176: break;
3177:
3178: pa = va_arg(ap, paddr_t);
3179: size = va_arg(ap, size_t);
3180:
3181: for (; va < (va + size); va += 0x1000, pa += 0x1000) {
3182: #if 0
1.48.30.1! bouyer 3183: printf("%s: Inserting: va: 0x%08lx, pa: 0x%08lx\n", __func__, va, pa);
1.38 sanjayl 3184: #endif
3185: ptegidx = va_to_pteg(pmap_kernel(), va);
3186: pmap_pte_create(&pte, pmap_kernel(), va, pa | pte_lo);
3187: i = pmap_pte_insert(ptegidx, &pte);
3188: }
3189: }
3190:
3191: TLBSYNC();
3192: SYNC();
3193: return (0);
3194: }
3195: #endif /* PPC_OEA64_BRIDGE */
3196:
1.1 matt 3197: /*
3198: * This is not part of the defined PMAP interface and is specific to the
3199: * PowerPC architecture. This is called during initppc, before the system
3200: * is really initialized.
3201: */
3202: void
3203: pmap_bootstrap(paddr_t kernelstart, paddr_t kernelend)
3204: {
3205: struct mem_region *mp, tmp;
3206: paddr_t s, e;
3207: psize_t size;
3208: int i, j;
3209:
3210: /*
3211: * Get memory.
3212: */
3213: mem_regions(&mem, &avail);
3214: #if defined(DEBUG)
3215: if (pmapdebug & PMAPDEBUG_BOOT) {
3216: printf("pmap_bootstrap: memory configuration:\n");
3217: for (mp = mem; mp->size; mp++) {
3218: printf("pmap_bootstrap: mem start 0x%lx size 0x%lx\n",
3219: mp->start, mp->size);
3220: }
3221: for (mp = avail; mp->size; mp++) {
3222: printf("pmap_bootstrap: avail start 0x%lx size 0x%lx\n",
3223: mp->start, mp->size);
3224: }
3225: }
3226: #endif
3227:
3228: /*
3229: * Find out how much physical memory we have and in how many chunks.
3230: */
3231: for (mem_cnt = 0, mp = mem; mp->size; mp++) {
3232: if (mp->start >= pmap_memlimit)
3233: continue;
3234: if (mp->start + mp->size > pmap_memlimit) {
3235: size = pmap_memlimit - mp->start;
3236: physmem += btoc(size);
3237: } else {
3238: physmem += btoc(mp->size);
3239: }
3240: mem_cnt++;
3241: }
3242:
3243: /*
3244: * Count the number of available entries.
3245: */
3246: for (avail_cnt = 0, mp = avail; mp->size; mp++)
3247: avail_cnt++;
3248:
3249: /*
3250: * Page align all regions.
3251: */
3252: kernelstart = trunc_page(kernelstart);
3253: kernelend = round_page(kernelend);
3254: for (mp = avail, i = 0; i < avail_cnt; i++, mp++) {
3255: s = round_page(mp->start);
3256: mp->size -= (s - mp->start);
3257: mp->size = trunc_page(mp->size);
3258: mp->start = s;
3259: e = mp->start + mp->size;
3260:
3261: DPRINTFN(BOOT,
3262: ("pmap_bootstrap: b-avail[%d] start 0x%lx size 0x%lx\n",
3263: i, mp->start, mp->size));
3264:
3265: /*
3266: * Don't allow the end to run beyond our artificial limit
3267: */
3268: if (e > pmap_memlimit)
3269: e = pmap_memlimit;
3270:
3271: /*
3272: * Is this region empty or strange? skip it.
3273: */
3274: if (e <= s) {
3275: mp->start = 0;
3276: mp->size = 0;
3277: continue;
3278: }
3279:
3280: /*
3281: * Does this overlap the beginning of kernel?
3282: * Does extend past the end of the kernel?
3283: */
3284: else if (s < kernelstart && e > kernelstart) {
3285: if (e > kernelend) {
3286: avail[avail_cnt].start = kernelend;
3287: avail[avail_cnt].size = e - kernelend;
3288: avail_cnt++;
3289: }
3290: mp->size = kernelstart - s;
3291: }
3292: /*
3293: * Check whether this region overlaps the end of the kernel.
3294: */
3295: else if (s < kernelend && e > kernelend) {
3296: mp->start = kernelend;
3297: mp->size = e - kernelend;
3298: }
3299: /*
3300: * Look whether this regions is completely inside the kernel.
3301: * Nuke it if it does.
3302: */
3303: else if (s >= kernelstart && e <= kernelend) {
3304: mp->start = 0;
3305: mp->size = 0;
3306: }
3307: /*
3308: * If the user imposed a memory limit, enforce it.
3309: */
3310: else if (s >= pmap_memlimit) {
1.6 thorpej 3311: mp->start = -PAGE_SIZE; /* let's know why */
1.1 matt 3312: mp->size = 0;
3313: }
3314: else {
3315: mp->start = s;
3316: mp->size = e - s;
3317: }
3318: DPRINTFN(BOOT,
3319: ("pmap_bootstrap: a-avail[%d] start 0x%lx size 0x%lx\n",
3320: i, mp->start, mp->size));
3321: }
3322:
3323: /*
3324: * Move (and uncount) all the null return to the end.
3325: */
3326: for (mp = avail, i = 0; i < avail_cnt; i++, mp++) {
3327: if (mp->size == 0) {
3328: tmp = avail[i];
3329: avail[i] = avail[--avail_cnt];
3330: avail[avail_cnt] = avail[i];
3331: }
3332: }
3333:
3334: /*
3335: * (Bubble)sort them into asecnding order.
3336: */
3337: for (i = 0; i < avail_cnt; i++) {
3338: for (j = i + 1; j < avail_cnt; j++) {
3339: if (avail[i].start > avail[j].start) {
3340: tmp = avail[i];
3341: avail[i] = avail[j];
3342: avail[j] = tmp;
3343: }
3344: }
3345: }
3346:
3347: /*
3348: * Make sure they don't overlap.
3349: */
3350: for (mp = avail, i = 0; i < avail_cnt - 1; i++, mp++) {
3351: if (mp[0].start + mp[0].size > mp[1].start) {
3352: mp[0].size = mp[1].start - mp[0].start;
3353: }
3354: DPRINTFN(BOOT,
3355: ("pmap_bootstrap: avail[%d] start 0x%lx size 0x%lx\n",
3356: i, mp->start, mp->size));
3357: }
3358: DPRINTFN(BOOT,
3359: ("pmap_bootstrap: avail[%d] start 0x%lx size 0x%lx\n",
3360: i, mp->start, mp->size));
3361:
3362: #ifdef PTEGCOUNT
3363: pmap_pteg_cnt = PTEGCOUNT;
3364: #else /* PTEGCOUNT */
1.38 sanjayl 3365:
1.1 matt 3366: pmap_pteg_cnt = 0x1000;
3367:
3368: while (pmap_pteg_cnt < physmem)
3369: pmap_pteg_cnt <<= 1;
3370:
3371: pmap_pteg_cnt >>= 1;
3372: #endif /* PTEGCOUNT */
3373:
1.38 sanjayl 3374: #ifdef DEBUG
3375: DPRINTFN(BOOT,
3376: ("pmap_pteg_cnt: 0x%x\n", pmap_pteg_cnt));
3377: #endif
3378:
1.1 matt 3379: /*
3380: * Find suitably aligned memory for PTEG hash table.
3381: */
1.2 matt 3382: size = pmap_pteg_cnt * sizeof(struct pteg);
1.1 matt 3383: pmap_pteg_table = pmap_boot_find_memory(size, size, 0);
1.38 sanjayl 3384:
3385: #ifdef DEBUG
3386: DPRINTFN(BOOT,
3387: ("PTEG cnt: 0x%x HTAB size: 0x%08x bytes, address: %p\n", pmap_pteg_cnt, (unsigned int)size, pmap_pteg_table));
3388: #endif
3389:
3390:
1.1 matt 3391: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
3392: if ( (uintptr_t) pmap_pteg_table + size > SEGMENT_LENGTH)
3393: panic("pmap_bootstrap: pmap_pteg_table end (%p + %lx) > 256MB",
3394: pmap_pteg_table, size);
3395: #endif
3396:
1.32 he 3397: memset(__UNVOLATILE(pmap_pteg_table), 0,
3398: pmap_pteg_cnt * sizeof(struct pteg));
1.1 matt 3399: pmap_pteg_mask = pmap_pteg_cnt - 1;
3400:
3401: /*
3402: * We cannot do pmap_steal_memory here since UVM hasn't been loaded
3403: * with pages. So we just steal them before giving them to UVM.
3404: */
3405: size = sizeof(pmap_pvo_table[0]) * pmap_pteg_cnt;
1.6 thorpej 3406: pmap_pvo_table = pmap_boot_find_memory(size, PAGE_SIZE, 0);
1.1 matt 3407: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
3408: if ( (uintptr_t) pmap_pvo_table + size > SEGMENT_LENGTH)
3409: panic("pmap_bootstrap: pmap_pvo_table end (%p + %lx) > 256MB",
3410: pmap_pvo_table, size);
3411: #endif
3412:
3413: for (i = 0; i < pmap_pteg_cnt; i++)
3414: TAILQ_INIT(&pmap_pvo_table[i]);
3415:
3416: #ifndef MSGBUFADDR
3417: /*
3418: * Allocate msgbuf in high memory.
3419: */
1.6 thorpej 3420: msgbuf_paddr =
3421: (paddr_t) pmap_boot_find_memory(MSGBUFSIZE, PAGE_SIZE, 1);
1.1 matt 3422: #endif
3423:
3424: #ifdef __HAVE_PMAP_PHYSSEG
3425: {
3426: u_int npgs = 0;
3427: for (i = 0, mp = avail; i < avail_cnt; i++, mp++)
3428: npgs += btoc(mp->size);
3429: size = (sizeof(struct pvo_head) + 1) * npgs;
1.6 thorpej 3430: pmap_physseg.pvoh = pmap_boot_find_memory(size, PAGE_SIZE, 0);
1.1 matt 3431: pmap_physseg.attrs = (char *) &pmap_physseg.pvoh[npgs];
3432: #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
3433: if ((uintptr_t)pmap_physseg.pvoh + size > SEGMENT_LENGTH)
3434: panic("pmap_bootstrap: PVO list end (%p + %lx) > 256MB",
3435: pmap_physseg.pvoh, size);
3436: #endif
3437: }
3438: #endif
3439:
1.38 sanjayl 3440:
1.1 matt 3441: for (mp = avail, i = 0; i < avail_cnt; mp++, i++) {
3442: paddr_t pfstart = atop(mp->start);
3443: paddr_t pfend = atop(mp->start + mp->size);
3444: if (mp->size == 0)
3445: continue;
3446: if (mp->start + mp->size <= SEGMENT_LENGTH) {
3447: uvm_page_physload(pfstart, pfend, pfstart, pfend,
3448: VM_FREELIST_FIRST256);
3449: } else if (mp->start >= SEGMENT_LENGTH) {
3450: uvm_page_physload(pfstart, pfend, pfstart, pfend,
3451: VM_FREELIST_DEFAULT);
3452: } else {
3453: pfend = atop(SEGMENT_LENGTH);
3454: uvm_page_physload(pfstart, pfend, pfstart, pfend,
3455: VM_FREELIST_FIRST256);
3456: pfstart = atop(SEGMENT_LENGTH);
3457: pfend = atop(mp->start + mp->size);
3458: uvm_page_physload(pfstart, pfend, pfstart, pfend,
3459: VM_FREELIST_DEFAULT);
3460: }
3461: }
3462:
3463: /*
3464: * Make sure kernel vsid is allocated as well as VSID 0.
3465: */
3466: pmap_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS-1)) / VSID_NBPW]
3467: |= 1 << (KERNEL_VSIDBITS % VSID_NBPW);
3468: pmap_vsid_bitmap[0] |= 1;
3469:
3470: /*
3471: * Initialize kernel pmap and hardware.
3472: */
1.38 sanjayl 3473:
3474: /* PPC_OEA64_BRIDGE does support these instructions */
3475: #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE)
1.1 matt 3476: for (i = 0; i < 16; i++) {
1.38 sanjayl 3477: pmap_kernel()->pm_sr[i] = KERNELN_SEGMENT(i)|SR_PRKEY;
1.35 perry 3478: __asm volatile ("mtsrin %0,%1"
1.38 sanjayl 3479: :: "r"(KERNELN_SEGMENT(i)|SR_PRKEY), "r"(i << ADDR_SR_SHFT));
1.1 matt 3480: }
3481:
3482: pmap_kernel()->pm_sr[KERNEL_SR] = KERNEL_SEGMENT|SR_SUKEY|SR_PRKEY;
1.35 perry 3483: __asm volatile ("mtsr %0,%1"
1.1 matt 3484: :: "n"(KERNEL_SR), "r"(KERNEL_SEGMENT));
3485: #ifdef KERNEL2_SR
3486: pmap_kernel()->pm_sr[KERNEL2_SR] = KERNEL2_SEGMENT|SR_SUKEY|SR_PRKEY;
1.35 perry 3487: __asm volatile ("mtsr %0,%1"
1.1 matt 3488: :: "n"(KERNEL2_SR), "r"(KERNEL2_SEGMENT));
3489: #endif
3490: for (i = 0; i < 16; i++) {
3491: if (iosrtable[i] & SR601_T) {
3492: pmap_kernel()->pm_sr[i] = iosrtable[i];
1.35 perry 3493: __asm volatile ("mtsrin %0,%1"
1.1 matt 3494: :: "r"(iosrtable[i]), "r"(i << ADDR_SR_SHFT));
3495: }
3496: }
1.38 sanjayl 3497: #endif /* PPC_OEA || PPC_OEA64_BRIDGE */
3498: #if defined (PPC_OEA)
1.35 perry 3499: __asm volatile ("sync; mtsdr1 %0; isync"
1.2 matt 3500: :: "r"((uintptr_t)pmap_pteg_table | (pmap_pteg_mask >> 10)));
1.38 sanjayl 3501: #elif defined (PPC_OEA64) || defined (PPC_OEA64_BRIDGE)
3502: __asm __volatile ("sync; mtsdr1 %0; isync"
3503: :: "r"((uintptr_t)pmap_pteg_table | (32 - cntlzw(pmap_pteg_mask >> 11))));
3504: #endif
1.1 matt 3505: tlbia();
3506:
3507: #ifdef ALTIVEC
3508: pmap_use_altivec = cpu_altivec;
3509: #endif
3510:
3511: #ifdef DEBUG
3512: if (pmapdebug & PMAPDEBUG_BOOT) {
3513: u_int cnt;
3514: int bank;
3515: char pbuf[9];
3516: for (cnt = 0, bank = 0; bank < vm_nphysseg; bank++) {
3517: cnt += vm_physmem[bank].avail_end - vm_physmem[bank].avail_start;
3518: printf("pmap_bootstrap: vm_physmem[%d]=%#lx-%#lx/%#lx\n",
3519: bank,
3520: ptoa(vm_physmem[bank].avail_start),
3521: ptoa(vm_physmem[bank].avail_end),
3522: ptoa(vm_physmem[bank].avail_end - vm_physmem[bank].avail_start));
3523: }
3524: format_bytes(pbuf, sizeof(pbuf), ptoa((u_int64_t) cnt));
3525: printf("pmap_bootstrap: UVM memory = %s (%u pages)\n",
3526: pbuf, cnt);
3527: }
3528: #endif
3529:
3530: pool_init(&pmap_upvo_pool, sizeof(struct pvo_entry),
3531: sizeof(struct pvo_entry), 0, 0, "pmap_upvopl",
1.48 ad 3532: &pmap_pool_uallocator, IPL_NONE);
1.1 matt 3533:
3534: pool_setlowat(&pmap_upvo_pool, 252);
3535:
3536: pool_init(&pmap_pool, sizeof(struct pmap),
1.48 ad 3537: sizeof(void *), 0, 0, "pmap_pl", &pmap_pool_uallocator,
3538: IPL_NONE);
1.41 matt 3539:
3540: #if defined(PMAP_NEED_MAPKERNEL)
3541: {
3542: extern int etext[], kernel_text[];
3543: vaddr_t va, va_etext = (paddr_t) etext;
3544: paddr_t pa;
1.42 matt 3545: register_t sr;
3546:
3547: sr = KERNELN_SEGMENT(kernelstart >> ADDR_SR_SHFT)
3548: |SR_SUKEY|SR_PRKEY;
1.41 matt 3549:
3550: va = (vaddr_t) kernel_text;
3551:
3552: for (pa = kernelstart; va < va_etext;
3553: pa += PAGE_SIZE, va += PAGE_SIZE)
3554: pmap_enter(pmap_kernel(), va, pa,
3555: VM_PROT_READ|VM_PROT_EXECUTE, 0);
3556:
3557: for (; pa < kernelend;
3558: pa += PAGE_SIZE, va += PAGE_SIZE)
3559: pmap_enter(pmap_kernel(), va, pa,
3560: VM_PROT_READ|VM_PROT_WRITE, 0);
1.42 matt 3561:
3562: pmap_kernel()->pm_sr[kernelstart >> ADDR_SR_SHFT] = sr;
3563: __asm volatile ("mtsrin %0,%1"
3564: :: "r"(sr), "r"(kernelstart));
1.41 matt 3565: }
3566: #endif
1.1 matt 3567: }
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to pmap.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / powerpc / oea