Annotation of src/sys/arch/arm/arm32/pmap.c, Revision 1.163.4.2
1.163.4.2! garbled 1: /* $NetBSD: pmap.c,v 1.165 2007/09/15 09:25:21 scw Exp $ */
1.12 chris 2:
3: /*
1.134 thorpej 4: * Copyright 2003 Wasabi Systems, Inc.
5: * All rights reserved.
6: *
7: * Written by Steve C. Woodford for Wasabi Systems, Inc.
8: *
9: * Redistribution and use in source and binary forms, with or without
10: * modification, are permitted provided that the following conditions
11: * are met:
12: * 1. Redistributions of source code must retain the above copyright
13: * notice, this list of conditions and the following disclaimer.
14: * 2. Redistributions in binary form must reproduce the above copyright
15: * notice, this list of conditions and the following disclaimer in the
16: * documentation and/or other materials provided with the distribution.
17: * 3. All advertising materials mentioning features or use of this software
18: * must display the following acknowledgement:
19: * This product includes software developed for the NetBSD Project by
20: * Wasabi Systems, Inc.
21: * 4. The name of Wasabi Systems, Inc. may not be used to endorse
22: * or promote products derived from this software without specific prior
23: * written permission.
24: *
25: * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
26: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
29: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35: * POSSIBILITY OF SUCH DAMAGE.
36: */
37:
38: /*
39: * Copyright (c) 2002-2003 Wasabi Systems, Inc.
1.12 chris 40: * Copyright (c) 2001 Richard Earnshaw
1.119 chris 41: * Copyright (c) 2001-2002 Christopher Gilbert
1.12 chris 42: * All rights reserved.
43: *
44: * 1. Redistributions of source code must retain the above copyright
45: * notice, this list of conditions and the following disclaimer.
46: * 2. Redistributions in binary form must reproduce the above copyright
47: * notice, this list of conditions and the following disclaimer in the
48: * documentation and/or other materials provided with the distribution.
49: * 3. The name of the company nor the name of the author may be used to
50: * endorse or promote products derived from this software without specific
51: * prior written permission.
52: *
53: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
54: * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
55: * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
56: * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
57: * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
58: * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
59: * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63: * SUCH DAMAGE.
64: */
1.1 matt 65:
66: /*-
67: * Copyright (c) 1999 The NetBSD Foundation, Inc.
68: * All rights reserved.
69: *
70: * This code is derived from software contributed to The NetBSD Foundation
71: * by Charles M. Hannum.
72: *
73: * Redistribution and use in source and binary forms, with or without
74: * modification, are permitted provided that the following conditions
75: * are met:
76: * 1. Redistributions of source code must retain the above copyright
77: * notice, this list of conditions and the following disclaimer.
78: * 2. Redistributions in binary form must reproduce the above copyright
79: * notice, this list of conditions and the following disclaimer in the
80: * documentation and/or other materials provided with the distribution.
81: * 3. All advertising materials mentioning features or use of this software
82: * must display the following acknowledgement:
83: * This product includes software developed by the NetBSD
84: * Foundation, Inc. and its contributors.
85: * 4. Neither the name of The NetBSD Foundation nor the names of its
86: * contributors may be used to endorse or promote products derived
87: * from this software without specific prior written permission.
88: *
89: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
90: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
91: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
92: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
93: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
94: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
95: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
96: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
97: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
98: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
99: * POSSIBILITY OF SUCH DAMAGE.
100: */
101:
102: /*
103: * Copyright (c) 1994-1998 Mark Brinicombe.
104: * Copyright (c) 1994 Brini.
105: * All rights reserved.
106: *
107: * This code is derived from software written for Brini by Mark Brinicombe
108: *
109: * Redistribution and use in source and binary forms, with or without
110: * modification, are permitted provided that the following conditions
111: * are met:
112: * 1. Redistributions of source code must retain the above copyright
113: * notice, this list of conditions and the following disclaimer.
114: * 2. Redistributions in binary form must reproduce the above copyright
115: * notice, this list of conditions and the following disclaimer in the
116: * documentation and/or other materials provided with the distribution.
117: * 3. All advertising materials mentioning features or use of this software
118: * must display the following acknowledgement:
119: * This product includes software developed by Mark Brinicombe.
120: * 4. The name of the author may not be used to endorse or promote products
121: * derived from this software without specific prior written permission.
122: *
123: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
124: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
125: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
126: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
127: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
128: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
129: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
130: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
131: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
132: *
133: * RiscBSD kernel project
134: *
135: * pmap.c
136: *
137: * Machine dependant vm stuff
138: *
139: * Created : 20/09/94
140: */
141:
142: /*
143: * Performance improvements, UVM changes, overhauls and part-rewrites
144: * were contributed by Neil A. Carson <neil@causality.com>.
145: */
146:
147: /*
1.134 thorpej 148: * Overhauled again to speedup the pmap, use MMU Domains so that L1 tables
149: * can be shared, and re-work the KVM layout, by Steve Woodford of Wasabi
150: * Systems, Inc.
151: *
152: * There are still a few things outstanding at this time:
153: *
154: * - There are some unresolved issues for MP systems:
155: *
156: * o The L1 metadata needs a lock, or more specifically, some places
157: * need to acquire an exclusive lock when modifying L1 translation
158: * table entries.
159: *
160: * o When one cpu modifies an L1 entry, and that L1 table is also
161: * being used by another cpu, then the latter will need to be told
162: * that a tlb invalidation may be necessary. (But only if the old
163: * domain number in the L1 entry being over-written is currently
164: * the active domain on that cpu). I guess there are lots more tlb
165: * shootdown issues too...
166: *
167: * o If the vector_page is at 0x00000000 instead of 0xffff0000, then
168: * MP systems will lose big-time because of the MMU domain hack.
169: * The only way this can be solved (apart from moving the vector
170: * page to 0xffff0000) is to reserve the first 1MB of user address
171: * space for kernel use only. This would require re-linking all
172: * applications so that the text section starts above this 1MB
173: * boundary.
174: *
175: * o Tracking which VM space is resident in the cache/tlb has not yet
176: * been implemented for MP systems.
177: *
178: * o Finally, there is a pathological condition where two cpus running
179: * two separate processes (not lwps) which happen to share an L1
180: * can get into a fight over one or more L1 entries. This will result
181: * in a significant slow-down if both processes are in tight loops.
1.1 matt 182: */
183:
184: /*
185: * Special compilation symbols
186: * PMAP_DEBUG - Build in pmap_debug_level code
187: */
1.134 thorpej 188:
1.1 matt 189: /* Include header files */
190:
1.134 thorpej 191: #include "opt_cpuoptions.h"
1.1 matt 192: #include "opt_pmap_debug.h"
193: #include "opt_ddb.h"
1.137 martin 194: #include "opt_lockdebug.h"
195: #include "opt_multiprocessor.h"
1.1 matt 196:
197: #include <sys/types.h>
198: #include <sys/param.h>
199: #include <sys/kernel.h>
200: #include <sys/systm.h>
201: #include <sys/proc.h>
202: #include <sys/malloc.h>
203: #include <sys/user.h>
1.10 chris 204: #include <sys/pool.h>
1.16 chris 205: #include <sys/cdefs.h>
206:
1.1 matt 207: #include <uvm/uvm.h>
208:
209: #include <machine/bus.h>
210: #include <machine/pmap.h>
211: #include <machine/pcb.h>
212: #include <machine/param.h>
1.32 thorpej 213: #include <arm/arm32/katelib.h>
1.16 chris 214:
1.163.4.2! garbled 215: __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.165 2007/09/15 09:25:21 scw Exp $");
1.116 jdolecek 216:
1.1 matt 217: #ifdef PMAP_DEBUG
1.140 matt 218:
219: /* XXX need to get rid of all refs to this */
1.134 thorpej 220: int pmap_debug_level = 0;
1.17 chris 221:
222: /*
223: * for switching to potentially finer grained debugging
224: */
225: #define PDB_FOLLOW 0x0001
226: #define PDB_INIT 0x0002
227: #define PDB_ENTER 0x0004
228: #define PDB_REMOVE 0x0008
229: #define PDB_CREATE 0x0010
230: #define PDB_PTPAGE 0x0020
1.48 chris 231: #define PDB_GROWKERN 0x0040
1.17 chris 232: #define PDB_BITS 0x0080
233: #define PDB_COLLECT 0x0100
234: #define PDB_PROTECT 0x0200
1.48 chris 235: #define PDB_MAP_L1 0x0400
1.17 chris 236: #define PDB_BOOTSTRAP 0x1000
237: #define PDB_PARANOIA 0x2000
238: #define PDB_WIRING 0x4000
239: #define PDB_PVDUMP 0x8000
1.134 thorpej 240: #define PDB_VAC 0x10000
241: #define PDB_KENTER 0x20000
242: #define PDB_KREMOVE 0x40000
1.17 chris 243:
1.134 thorpej 244: int debugmap = 1;
245: int pmapdebug = 0;
1.17 chris 246: #define NPDEBUG(_lev_,_stat_) \
247: if (pmapdebug & (_lev_)) \
248: ((_stat_))
249:
1.1 matt 250: #else /* PMAP_DEBUG */
1.48 chris 251: #define NPDEBUG(_lev_,_stat_) /* Nothing */
1.1 matt 252: #endif /* PMAP_DEBUG */
253:
1.134 thorpej 254: /*
255: * pmap_kernel() points here
256: */
1.1 matt 257: struct pmap kernel_pmap_store;
258:
1.10 chris 259: /*
1.134 thorpej 260: * Which pmap is currently 'live' in the cache
261: *
262: * XXXSCW: Fix for SMP ...
1.48 chris 263: */
1.163.4.2! garbled 264: static pmap_t pmap_recent_user;
1.48 chris 265:
1.134 thorpej 266: /*
267: * Pool and cache that pmap structures are allocated from.
268: * We use a cache to avoid clearing the pm_l2[] array (1KB)
269: * in pmap_create().
270: */
271: static struct pool pmap_pmap_pool;
272: static struct pool_cache pmap_pmap_cache;
273: static LIST_HEAD(, pmap) pmap_pmaps;
1.48 chris 274:
275: /*
1.134 thorpej 276: * Pool of PV structures
1.10 chris 277: */
1.134 thorpej 278: static struct pool pmap_pv_pool;
279: static void *pmap_bootstrap_pv_page_alloc(struct pool *, int);
280: static void pmap_bootstrap_pv_page_free(struct pool *, void *);
281: static struct pool_allocator pmap_bootstrap_pv_allocator = {
282: pmap_bootstrap_pv_page_alloc, pmap_bootstrap_pv_page_free
283: };
1.10 chris 284:
1.134 thorpej 285: /*
286: * Pool and cache of l2_dtable structures.
287: * We use a cache to avoid clearing the structures when they're
288: * allocated. (196 bytes)
289: */
290: static struct pool pmap_l2dtable_pool;
291: static struct pool_cache pmap_l2dtable_cache;
292: static vaddr_t pmap_kernel_l2dtable_kva;
1.10 chris 293:
1.111 thorpej 294: /*
1.134 thorpej 295: * Pool and cache of L2 page descriptors.
296: * We use a cache to avoid clearing the descriptor table
297: * when they're allocated. (1KB)
1.111 thorpej 298: */
1.134 thorpej 299: static struct pool pmap_l2ptp_pool;
300: static struct pool_cache pmap_l2ptp_cache;
301: static vaddr_t pmap_kernel_l2ptp_kva;
302: static paddr_t pmap_kernel_l2ptp_phys;
1.111 thorpej 303:
1.134 thorpej 304: /*
305: * pmap copy/zero page, and mem(5) hook point
306: */
1.54 thorpej 307: static pt_entry_t *csrc_pte, *cdst_pte;
308: static vaddr_t csrcp, cdstp;
1.1 matt 309: char *memhook;
1.161 christos 310: extern void *msgbufaddr;
1.1 matt 311:
1.17 chris 312: /*
1.134 thorpej 313: * Flag to indicate if pmap_init() has done its thing
314: */
1.159 thorpej 315: bool pmap_initialized;
1.134 thorpej 316:
317: /*
318: * Misc. locking data structures
1.17 chris 319: */
1.1 matt 320:
1.134 thorpej 321: #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
1.17 chris 322: static struct lock pmap_main_lock;
1.134 thorpej 323:
1.17 chris 324: #define PMAP_MAP_TO_HEAD_LOCK() \
325: (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL)
326: #define PMAP_MAP_TO_HEAD_UNLOCK() \
327: (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL)
328: #define PMAP_HEAD_TO_MAP_LOCK() \
329: (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL)
330: #define PMAP_HEAD_TO_MAP_UNLOCK() \
1.134 thorpej 331: spinlockmgr(&pmap_main_lock, LK_RELEASE, (void *) 0)
1.17 chris 332: #else
1.134 thorpej 333: #define PMAP_MAP_TO_HEAD_LOCK() /* null */
334: #define PMAP_MAP_TO_HEAD_UNLOCK() /* null */
335: #define PMAP_HEAD_TO_MAP_LOCK() /* null */
336: #define PMAP_HEAD_TO_MAP_UNLOCK() /* null */
337: #endif
338:
339: #define pmap_acquire_pmap_lock(pm) \
340: do { \
341: if ((pm) != pmap_kernel()) \
342: simple_lock(&(pm)->pm_lock); \
343: } while (/*CONSTCOND*/0)
344:
345: #define pmap_release_pmap_lock(pm) \
346: do { \
347: if ((pm) != pmap_kernel()) \
348: simple_unlock(&(pm)->pm_lock); \
349: } while (/*CONSTCOND*/0)
1.1 matt 350:
1.33 chris 351:
1.69 thorpej 352: /*
1.134 thorpej 353: * Metadata for L1 translation tables.
1.69 thorpej 354: */
1.134 thorpej 355: struct l1_ttable {
356: /* Entry on the L1 Table list */
357: SLIST_ENTRY(l1_ttable) l1_link;
1.1 matt 358:
1.134 thorpej 359: /* Entry on the L1 Least Recently Used list */
360: TAILQ_ENTRY(l1_ttable) l1_lru;
1.1 matt 361:
1.134 thorpej 362: /* Track how many domains are allocated from this L1 */
363: volatile u_int l1_domain_use_count;
1.1 matt 364:
1.134 thorpej 365: /*
366: * A free-list of domain numbers for this L1.
367: * We avoid using ffs() and a bitmap to track domains since ffs()
368: * is slow on ARM.
369: */
370: u_int8_t l1_domain_first;
371: u_int8_t l1_domain_free[PMAP_DOMAINS];
1.1 matt 372:
1.134 thorpej 373: /* Physical address of this L1 page table */
374: paddr_t l1_physaddr;
1.1 matt 375:
1.134 thorpej 376: /* KVA of this L1 page table */
377: pd_entry_t *l1_kva;
378: };
1.1 matt 379:
1.134 thorpej 380: /*
381: * Convert a virtual address into its L1 table index. That is, the
382: * index used to locate the L2 descriptor table pointer in an L1 table.
383: * This is basically used to index l1->l1_kva[].
384: *
385: * Each L2 descriptor table represents 1MB of VA space.
386: */
387: #define L1_IDX(va) (((vaddr_t)(va)) >> L1_S_SHIFT)
1.11 chris 388:
1.17 chris 389: /*
1.134 thorpej 390: * L1 Page Tables are tracked using a Least Recently Used list.
391: * - New L1s are allocated from the HEAD.
392: * - Freed L1s are added to the TAIl.
393: * - Recently accessed L1s (where an 'access' is some change to one of
394: * the userland pmaps which owns this L1) are moved to the TAIL.
1.17 chris 395: */
1.134 thorpej 396: static TAILQ_HEAD(, l1_ttable) l1_lru_list;
397: static struct simplelock l1_lru_lock;
1.17 chris 398:
1.134 thorpej 399: /*
400: * A list of all L1 tables
401: */
402: static SLIST_HEAD(, l1_ttable) l1_list;
1.17 chris 403:
404: /*
1.134 thorpej 405: * The l2_dtable tracks L2_BUCKET_SIZE worth of L1 slots.
406: *
407: * This is normally 16MB worth L2 page descriptors for any given pmap.
408: * Reference counts are maintained for L2 descriptors so they can be
409: * freed when empty.
1.17 chris 410: */
1.134 thorpej 411: struct l2_dtable {
412: /* The number of L2 page descriptors allocated to this l2_dtable */
413: u_int l2_occupancy;
1.17 chris 414:
1.134 thorpej 415: /* List of L2 page descriptors */
416: struct l2_bucket {
417: pt_entry_t *l2b_kva; /* KVA of L2 Descriptor Table */
418: paddr_t l2b_phys; /* Physical address of same */
419: u_short l2b_l1idx; /* This L2 table's L1 index */
420: u_short l2b_occupancy; /* How many active descriptors */
421: } l2_bucket[L2_BUCKET_SIZE];
1.17 chris 422: };
423:
424: /*
1.134 thorpej 425: * Given an L1 table index, calculate the corresponding l2_dtable index
426: * and bucket index within the l2_dtable.
1.17 chris 427: */
1.134 thorpej 428: #define L2_IDX(l1idx) (((l1idx) >> L2_BUCKET_LOG2) & \
429: (L2_SIZE - 1))
430: #define L2_BUCKET(l1idx) ((l1idx) & (L2_BUCKET_SIZE - 1))
1.17 chris 431:
1.134 thorpej 432: /*
433: * Given a virtual address, this macro returns the
434: * virtual address required to drop into the next L2 bucket.
435: */
436: #define L2_NEXT_BUCKET(va) (((va) & L1_S_FRAME) + L1_S_SIZE)
1.17 chris 437:
438: /*
1.134 thorpej 439: * L2 allocation.
1.17 chris 440: */
1.134 thorpej 441: #define pmap_alloc_l2_dtable() \
442: pool_cache_get(&pmap_l2dtable_cache, PR_NOWAIT)
443: #define pmap_free_l2_dtable(l2) \
444: pool_cache_put(&pmap_l2dtable_cache, (l2))
445: #define pmap_alloc_l2_ptp(pap) \
446: ((pt_entry_t *)pool_cache_get_paddr(&pmap_l2ptp_cache,\
447: PR_NOWAIT, (pap)))
1.1 matt 448:
449: /*
1.134 thorpej 450: * We try to map the page tables write-through, if possible. However, not
451: * all CPUs have a write-through cache mode, so on those we have to sync
452: * the cache when we frob page tables.
1.113 thorpej 453: *
1.134 thorpej 454: * We try to evaluate this at compile time, if possible. However, it's
455: * not always possible to do that, hence this run-time var.
456: */
457: int pmap_needs_pte_sync;
1.113 thorpej 458:
459: /*
1.134 thorpej 460: * Real definition of pv_entry.
1.113 thorpej 461: */
1.134 thorpej 462: struct pv_entry {
463: struct pv_entry *pv_next; /* next pv_entry */
464: pmap_t pv_pmap; /* pmap where mapping lies */
465: vaddr_t pv_va; /* virtual address for mapping */
466: u_int pv_flags; /* flags */
467: };
1.113 thorpej 468:
469: /*
1.134 thorpej 470: * Macro to determine if a mapping might be resident in the
471: * instruction cache and/or TLB
1.17 chris 472: */
1.134 thorpej 473: #define PV_BEEN_EXECD(f) (((f) & (PVF_REF | PVF_EXEC)) == (PVF_REF | PVF_EXEC))
1.17 chris 474:
475: /*
1.134 thorpej 476: * Macro to determine if a mapping might be resident in the
477: * data cache and/or TLB
1.1 matt 478: */
1.134 thorpej 479: #define PV_BEEN_REFD(f) (((f) & PVF_REF) != 0)
1.1 matt 480:
481: /*
1.134 thorpej 482: * Local prototypes
1.1 matt 483: */
1.134 thorpej 484: static int pmap_set_pt_cache_mode(pd_entry_t *, vaddr_t);
485: static void pmap_alloc_specials(vaddr_t *, int, vaddr_t *,
486: pt_entry_t **);
1.159 thorpej 487: static bool pmap_is_current(pmap_t);
488: static bool pmap_is_cached(pmap_t);
1.134 thorpej 489: static void pmap_enter_pv(struct vm_page *, struct pv_entry *,
490: pmap_t, vaddr_t, u_int);
491: static struct pv_entry *pmap_find_pv(struct vm_page *, pmap_t, vaddr_t);
1.156 scw 492: static struct pv_entry *pmap_remove_pv(struct vm_page *, pmap_t, vaddr_t, int);
1.134 thorpej 493: static u_int pmap_modify_pv(struct vm_page *, pmap_t, vaddr_t,
494: u_int, u_int);
1.17 chris 495:
1.134 thorpej 496: static void pmap_pinit(pmap_t);
497: static int pmap_pmap_ctor(void *, void *, int);
1.17 chris 498:
1.134 thorpej 499: static void pmap_alloc_l1(pmap_t);
500: static void pmap_free_l1(pmap_t);
501: static void pmap_use_l1(pmap_t);
1.17 chris 502:
1.134 thorpej 503: static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vaddr_t);
504: static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vaddr_t);
505: static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int);
506: static int pmap_l2ptp_ctor(void *, void *, int);
507: static int pmap_l2dtable_ctor(void *, void *, int);
1.51 chris 508:
1.134 thorpej 509: static void pmap_vac_me_harder(struct vm_page *, pmap_t, vaddr_t);
510: static void pmap_vac_me_kpmap(struct vm_page *, pmap_t, vaddr_t);
511: static void pmap_vac_me_user(struct vm_page *, pmap_t, vaddr_t);
1.17 chris 512:
1.134 thorpej 513: static void pmap_clearbit(struct vm_page *, u_int);
1.159 thorpej 514: static int pmap_clean_page(struct pv_entry *, bool);
1.134 thorpej 515: static void pmap_page_remove(struct vm_page *);
1.17 chris 516:
1.134 thorpej 517: static void pmap_init_l1(struct l1_ttable *, pd_entry_t *);
518: static vaddr_t kernel_pt_lookup(paddr_t);
1.17 chris 519:
1.163.4.2! garbled 520: void pmap_switch(struct lwp *, struct lwp *);
! 521:
1.17 chris 522:
523: /*
1.134 thorpej 524: * External function prototypes
1.17 chris 525: */
1.134 thorpej 526: extern void bzero_page(vaddr_t);
527: extern void bcopy_page(vaddr_t, vaddr_t);
1.17 chris 528:
1.134 thorpej 529: /*
530: * Misc variables
531: */
532: vaddr_t virtual_avail;
533: vaddr_t virtual_end;
534: vaddr_t pmap_curmaxkvaddr;
1.17 chris 535:
1.134 thorpej 536: vaddr_t avail_start;
537: vaddr_t avail_end;
1.17 chris 538:
1.134 thorpej 539: extern pv_addr_t systempage;
1.17 chris 540:
1.134 thorpej 541: /* Function to set the debug level of the pmap code */
1.17 chris 542:
1.134 thorpej 543: #ifdef PMAP_DEBUG
544: void
545: pmap_debug(int level)
546: {
547: pmap_debug_level = level;
548: printf("pmap_debug: level=%d\n", pmap_debug_level);
1.1 matt 549: }
1.134 thorpej 550: #endif /* PMAP_DEBUG */
1.1 matt 551:
552: /*
1.134 thorpej 553: * A bunch of routines to conditionally flush the caches/TLB depending
554: * on whether the specified pmap actually needs to be flushed at any
555: * given time.
1.1 matt 556: */
1.157 perry 557: static inline void
1.134 thorpej 558: pmap_tlb_flushID_SE(pmap_t pm, vaddr_t va)
559: {
560:
561: if (pm->pm_cstate.cs_tlb_id)
562: cpu_tlb_flushID_SE(va);
563: }
1.1 matt 564:
1.157 perry 565: static inline void
1.134 thorpej 566: pmap_tlb_flushD_SE(pmap_t pm, vaddr_t va)
1.1 matt 567: {
568:
1.134 thorpej 569: if (pm->pm_cstate.cs_tlb_d)
570: cpu_tlb_flushD_SE(va);
1.1 matt 571: }
572:
1.157 perry 573: static inline void
1.134 thorpej 574: pmap_tlb_flushID(pmap_t pm)
1.1 matt 575: {
576:
1.134 thorpej 577: if (pm->pm_cstate.cs_tlb_id) {
578: cpu_tlb_flushID();
579: pm->pm_cstate.cs_tlb = 0;
1.1 matt 580: }
1.134 thorpej 581: }
1.1 matt 582:
1.157 perry 583: static inline void
1.134 thorpej 584: pmap_tlb_flushD(pmap_t pm)
585: {
1.1 matt 586:
1.134 thorpej 587: if (pm->pm_cstate.cs_tlb_d) {
588: cpu_tlb_flushD();
589: pm->pm_cstate.cs_tlb_d = 0;
1.1 matt 590: }
591: }
592:
1.157 perry 593: static inline void
1.134 thorpej 594: pmap_idcache_wbinv_range(pmap_t pm, vaddr_t va, vsize_t len)
1.1 matt 595: {
1.17 chris 596:
1.134 thorpej 597: if (pm->pm_cstate.cs_cache_id)
598: cpu_idcache_wbinv_range(va, len);
1.17 chris 599: }
1.1 matt 600:
1.157 perry 601: static inline void
1.134 thorpej 602: pmap_dcache_wb_range(pmap_t pm, vaddr_t va, vsize_t len,
1.159 thorpej 603: bool do_inv, bool rd_only)
1.17 chris 604: {
1.1 matt 605:
1.134 thorpej 606: if (pm->pm_cstate.cs_cache_d) {
607: if (do_inv) {
608: if (rd_only)
609: cpu_dcache_inv_range(va, len);
610: else
611: cpu_dcache_wbinv_range(va, len);
612: } else
613: if (!rd_only)
614: cpu_dcache_wb_range(va, len);
1.1 matt 615: }
1.134 thorpej 616: }
1.1 matt 617:
1.157 perry 618: static inline void
1.134 thorpej 619: pmap_idcache_wbinv_all(pmap_t pm)
620: {
1.1 matt 621:
1.134 thorpej 622: if (pm->pm_cstate.cs_cache_id) {
623: cpu_idcache_wbinv_all();
624: pm->pm_cstate.cs_cache = 0;
625: }
1.1 matt 626: }
627:
1.157 perry 628: static inline void
1.134 thorpej 629: pmap_dcache_wbinv_all(pmap_t pm)
630: {
1.1 matt 631:
1.134 thorpej 632: if (pm->pm_cstate.cs_cache_d) {
633: cpu_dcache_wbinv_all();
634: pm->pm_cstate.cs_cache_d = 0;
635: }
636: }
1.1 matt 637:
1.159 thorpej 638: static inline bool
1.134 thorpej 639: pmap_is_current(pmap_t pm)
1.1 matt 640: {
1.17 chris 641:
1.134 thorpej 642: if (pm == pmap_kernel() ||
643: (curproc && curproc->p_vmspace->vm_map.pmap == pm))
1.160 thorpej 644: return (true);
1.1 matt 645:
1.160 thorpej 646: return (false);
1.134 thorpej 647: }
1.1 matt 648:
1.159 thorpej 649: static inline bool
1.134 thorpej 650: pmap_is_cached(pmap_t pm)
651: {
1.17 chris 652:
1.163.4.2! garbled 653: if (pm == pmap_kernel() || pmap_recent_user == NULL ||
! 654: pmap_recent_user == pm)
1.160 thorpej 655: return (true);
1.17 chris 656:
1.160 thorpej 657: return (false);
1.134 thorpej 658: }
1.1 matt 659:
1.134 thorpej 660: /*
661: * PTE_SYNC_CURRENT:
662: *
663: * Make sure the pte is written out to RAM.
664: * We need to do this for one of two cases:
665: * - We're dealing with the kernel pmap
666: * - There is no pmap active in the cache/tlb.
667: * - The specified pmap is 'active' in the cache/tlb.
668: */
669: #ifdef PMAP_INCLUDE_PTE_SYNC
670: #define PTE_SYNC_CURRENT(pm, ptep) \
671: do { \
672: if (PMAP_NEEDS_PTE_SYNC && \
673: pmap_is_cached(pm)) \
674: PTE_SYNC(ptep); \
675: } while (/*CONSTCOND*/0)
676: #else
677: #define PTE_SYNC_CURRENT(pm, ptep) /* nothing */
678: #endif
1.1 matt 679:
680: /*
1.17 chris 681: * main pv_entry manipulation functions:
1.49 thorpej 682: * pmap_enter_pv: enter a mapping onto a vm_page list
683: * pmap_remove_pv: remove a mappiing from a vm_page list
1.17 chris 684: *
685: * NOTE: pmap_enter_pv expects to lock the pvh itself
686: * pmap_remove_pv expects te caller to lock the pvh before calling
687: */
688:
689: /*
1.49 thorpej 690: * pmap_enter_pv: enter a mapping onto a vm_page lst
1.17 chris 691: *
692: * => caller should hold the proper lock on pmap_main_lock
693: * => caller should have pmap locked
1.49 thorpej 694: * => we will gain the lock on the vm_page and allocate the new pv_entry
1.17 chris 695: * => caller should adjust ptp's wire_count before calling
696: * => caller should not adjust pmap's wire_count
697: */
1.134 thorpej 698: static void
699: pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm,
700: vaddr_t va, u_int flags)
701: {
1.17 chris 702:
1.134 thorpej 703: NPDEBUG(PDB_PVDUMP,
704: printf("pmap_enter_pv: pm %p, pg %p, flags 0x%x\n", pm, pg, flags));
705:
706: pve->pv_pmap = pm;
1.17 chris 707: pve->pv_va = va;
708: pve->pv_flags = flags;
1.134 thorpej 709:
1.49 thorpej 710: simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */
711: pve->pv_next = pg->mdpage.pvh_list; /* add to ... */
712: pg->mdpage.pvh_list = pve; /* ... locked list */
1.134 thorpej 713: pg->mdpage.pvh_attrs |= flags & (PVF_REF | PVF_MOD);
714: if (pm == pmap_kernel()) {
715: if (flags & PVF_WRITE)
716: pg->mdpage.krw_mappings++;
717: else
718: pg->mdpage.kro_mappings++;
719: } else
720: if (flags & PVF_WRITE)
721: pg->mdpage.urw_mappings++;
722: else
723: pg->mdpage.uro_mappings++;
1.49 thorpej 724: simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */
1.134 thorpej 725:
1.78 thorpej 726: if (pve->pv_flags & PVF_WIRED)
1.134 thorpej 727: ++pm->pm_stats.wired_count;
1.17 chris 728: }
729:
730: /*
1.134 thorpej 731: *
732: * pmap_find_pv: Find a pv entry
733: *
734: * => caller should hold lock on vm_page
735: */
1.157 perry 736: static inline struct pv_entry *
1.134 thorpej 737: pmap_find_pv(struct vm_page *pg, pmap_t pm, vaddr_t va)
738: {
739: struct pv_entry *pv;
740:
741: for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) {
742: if (pm == pv->pv_pmap && va == pv->pv_va)
743: break;
744: }
745:
746: return (pv);
747: }
748:
749: /*
750: * pmap_remove_pv: try to remove a mapping from a pv_list
1.17 chris 751: *
752: * => caller should hold proper lock on pmap_main_lock
753: * => pmap should be locked
1.49 thorpej 754: * => caller should hold lock on vm_page [so that attrs can be adjusted]
1.17 chris 755: * => caller should adjust ptp's wire_count and free PTP if needed
756: * => caller should NOT adjust pmap's wire_count
757: * => we return the removed pve
758: */
1.134 thorpej 759: static struct pv_entry *
1.156 scw 760: pmap_remove_pv(struct vm_page *pg, pmap_t pm, vaddr_t va, int skip_wired)
1.17 chris 761: {
762: struct pv_entry *pve, **prevptr;
763:
1.134 thorpej 764: NPDEBUG(PDB_PVDUMP,
765: printf("pmap_remove_pv: pm %p, pg %p, va 0x%08lx\n", pm, pg, va));
766:
1.49 thorpej 767: prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */
1.17 chris 768: pve = *prevptr;
1.134 thorpej 769:
1.17 chris 770: while (pve) {
1.134 thorpej 771: if (pve->pv_pmap == pm && pve->pv_va == va) { /* match? */
1.156 scw 772: NPDEBUG(PDB_PVDUMP, printf("pmap_remove_pv: pm %p, pg "
773: "%p, flags 0x%x\n", pm, pg, pve->pv_flags));
774: if (pve->pv_flags & PVF_WIRED) {
775: if (skip_wired)
776: return (NULL);
777: --pm->pm_stats.wired_count;
778: }
1.17 chris 779: *prevptr = pve->pv_next; /* remove it! */
1.134 thorpej 780: if (pm == pmap_kernel()) {
781: if (pve->pv_flags & PVF_WRITE)
782: pg->mdpage.krw_mappings--;
783: else
784: pg->mdpage.kro_mappings--;
785: } else
786: if (pve->pv_flags & PVF_WRITE)
787: pg->mdpage.urw_mappings--;
788: else
789: pg->mdpage.uro_mappings--;
1.17 chris 790: break;
791: }
792: prevptr = &pve->pv_next; /* previous pointer */
793: pve = pve->pv_next; /* advance */
794: }
1.134 thorpej 795:
1.17 chris 796: return(pve); /* return removed pve */
797: }
798:
799: /*
800: *
801: * pmap_modify_pv: Update pv flags
802: *
1.49 thorpej 803: * => caller should hold lock on vm_page [so that attrs can be adjusted]
1.17 chris 804: * => caller should NOT adjust pmap's wire_count
1.29 rearnsha 805: * => caller must call pmap_vac_me_harder() if writable status of a page
806: * may have changed.
1.17 chris 807: * => we return the old flags
808: *
1.1 matt 809: * Modify a physical-virtual mapping in the pv table
810: */
1.134 thorpej 811: static u_int
812: pmap_modify_pv(struct vm_page *pg, pmap_t pm, vaddr_t va,
813: u_int clr_mask, u_int set_mask)
1.1 matt 814: {
815: struct pv_entry *npv;
816: u_int flags, oflags;
817:
1.134 thorpej 818: if ((npv = pmap_find_pv(pg, pm, va)) == NULL)
819: return (0);
820:
821: NPDEBUG(PDB_PVDUMP,
822: printf("pmap_modify_pv: pm %p, pg %p, clr 0x%x, set 0x%x, flags 0x%x\n", pm, pg, clr_mask, set_mask, npv->pv_flags));
823:
1.1 matt 824: /*
825: * There is at least one VA mapping this page.
826: */
827:
1.134 thorpej 828: if (clr_mask & (PVF_REF | PVF_MOD))
829: pg->mdpage.pvh_attrs |= set_mask & (PVF_REF | PVF_MOD);
830:
831: oflags = npv->pv_flags;
832: npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask;
833:
834: if ((flags ^ oflags) & PVF_WIRED) {
835: if (flags & PVF_WIRED)
836: ++pm->pm_stats.wired_count;
837: else
838: --pm->pm_stats.wired_count;
839: }
840:
841: if ((flags ^ oflags) & PVF_WRITE) {
842: if (pm == pmap_kernel()) {
843: if (flags & PVF_WRITE) {
844: pg->mdpage.krw_mappings++;
845: pg->mdpage.kro_mappings--;
846: } else {
847: pg->mdpage.kro_mappings++;
848: pg->mdpage.krw_mappings--;
1.1 matt 849: }
1.134 thorpej 850: } else
851: if (flags & PVF_WRITE) {
852: pg->mdpage.urw_mappings++;
853: pg->mdpage.uro_mappings--;
854: } else {
855: pg->mdpage.uro_mappings++;
856: pg->mdpage.urw_mappings--;
1.1 matt 857: }
858: }
1.134 thorpej 859:
860: return (oflags);
1.1 matt 861: }
862:
1.134 thorpej 863: /*
864: * Allocate an L1 translation table for the specified pmap.
865: * This is called at pmap creation time.
866: */
867: static void
868: pmap_alloc_l1(pmap_t pm)
1.1 matt 869: {
1.134 thorpej 870: struct l1_ttable *l1;
871: u_int8_t domain;
872:
873: /*
874: * Remove the L1 at the head of the LRU list
875: */
876: simple_lock(&l1_lru_lock);
877: l1 = TAILQ_FIRST(&l1_lru_list);
878: KDASSERT(l1 != NULL);
879: TAILQ_REMOVE(&l1_lru_list, l1, l1_lru);
1.1 matt 880:
1.134 thorpej 881: /*
882: * Pick the first available domain number, and update
883: * the link to the next number.
884: */
885: domain = l1->l1_domain_first;
886: l1->l1_domain_first = l1->l1_domain_free[domain];
1.115 thorpej 887:
1.134 thorpej 888: /*
889: * If there are still free domain numbers in this L1,
890: * put it back on the TAIL of the LRU list.
891: */
892: if (++l1->l1_domain_use_count < PMAP_DOMAINS)
893: TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru);
1.1 matt 894:
1.134 thorpej 895: simple_unlock(&l1_lru_lock);
1.1 matt 896:
1.134 thorpej 897: /*
898: * Fix up the relevant bits in the pmap structure
899: */
900: pm->pm_l1 = l1;
901: pm->pm_domain = domain;
1.1 matt 902: }
903:
904: /*
1.134 thorpej 905: * Free an L1 translation table.
906: * This is called at pmap destruction time.
1.1 matt 907: */
1.134 thorpej 908: static void
909: pmap_free_l1(pmap_t pm)
1.1 matt 910: {
1.134 thorpej 911: struct l1_ttable *l1 = pm->pm_l1;
1.1 matt 912:
1.134 thorpej 913: simple_lock(&l1_lru_lock);
1.1 matt 914:
1.134 thorpej 915: /*
916: * If this L1 is currently on the LRU list, remove it.
917: */
918: if (l1->l1_domain_use_count < PMAP_DOMAINS)
919: TAILQ_REMOVE(&l1_lru_list, l1, l1_lru);
1.1 matt 920:
921: /*
1.134 thorpej 922: * Free up the domain number which was allocated to the pmap
1.1 matt 923: */
1.134 thorpej 924: l1->l1_domain_free[pm->pm_domain] = l1->l1_domain_first;
925: l1->l1_domain_first = pm->pm_domain;
926: l1->l1_domain_use_count--;
1.1 matt 927:
1.134 thorpej 928: /*
929: * The L1 now must have at least 1 free domain, so add
930: * it back to the LRU list. If the use count is zero,
931: * put it at the head of the list, otherwise it goes
932: * to the tail.
933: */
934: if (l1->l1_domain_use_count == 0)
935: TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru);
936: else
937: TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru);
1.54 thorpej 938:
1.134 thorpej 939: simple_unlock(&l1_lru_lock);
940: }
1.54 thorpej 941:
1.157 perry 942: static inline void
1.134 thorpej 943: pmap_use_l1(pmap_t pm)
944: {
945: struct l1_ttable *l1;
1.54 thorpej 946:
1.134 thorpej 947: /*
948: * Do nothing if we're in interrupt context.
949: * Access to an L1 by the kernel pmap must not affect
950: * the LRU list.
951: */
952: if (current_intr_depth || pm == pmap_kernel())
953: return;
1.54 thorpej 954:
1.134 thorpej 955: l1 = pm->pm_l1;
1.1 matt 956:
1.17 chris 957: /*
1.134 thorpej 958: * If the L1 is not currently on the LRU list, just return
1.17 chris 959: */
1.134 thorpej 960: if (l1->l1_domain_use_count == PMAP_DOMAINS)
961: return;
962:
963: simple_lock(&l1_lru_lock);
1.1 matt 964:
1.10 chris 965: /*
1.134 thorpej 966: * Check the use count again, now that we've acquired the lock
1.10 chris 967: */
1.134 thorpej 968: if (l1->l1_domain_use_count == PMAP_DOMAINS) {
969: simple_unlock(&l1_lru_lock);
970: return;
971: }
1.111 thorpej 972:
973: /*
1.134 thorpej 974: * Move the L1 to the back of the LRU list
1.111 thorpej 975: */
1.134 thorpej 976: TAILQ_REMOVE(&l1_lru_list, l1, l1_lru);
977: TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru);
1.111 thorpej 978:
1.134 thorpej 979: simple_unlock(&l1_lru_lock);
1.1 matt 980: }
981:
982: /*
1.134 thorpej 983: * void pmap_free_l2_ptp(pt_entry_t *, paddr_t *)
1.1 matt 984: *
1.134 thorpej 985: * Free an L2 descriptor table.
1.1 matt 986: */
1.157 perry 987: static inline void
1.134 thorpej 988: #ifndef PMAP_INCLUDE_PTE_SYNC
989: pmap_free_l2_ptp(pt_entry_t *l2, paddr_t pa)
990: #else
1.159 thorpej 991: pmap_free_l2_ptp(bool need_sync, pt_entry_t *l2, paddr_t pa)
1.134 thorpej 992: #endif
1.1 matt 993: {
1.134 thorpej 994: #ifdef PMAP_INCLUDE_PTE_SYNC
1.1 matt 995: /*
1.134 thorpej 996: * Note: With a write-back cache, we may need to sync this
997: * L2 table before re-using it.
998: * This is because it may have belonged to a non-current
999: * pmap, in which case the cache syncs would have been
1000: * skipped when the pages were being unmapped. If the
1001: * L2 table were then to be immediately re-allocated to
1002: * the *current* pmap, it may well contain stale mappings
1003: * which have not yet been cleared by a cache write-back
1004: * and so would still be visible to the mmu.
1.1 matt 1005: */
1.134 thorpej 1006: if (need_sync)
1007: PTE_SYNC_RANGE(l2, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t));
1008: #endif
1009: pool_cache_put_paddr(&pmap_l2ptp_cache, (void *)l2, pa);
1.1 matt 1010: }
1011:
1012: /*
1.134 thorpej 1013: * Returns a pointer to the L2 bucket associated with the specified pmap
1014: * and VA, or NULL if no L2 bucket exists for the address.
1.1 matt 1015: */
1.157 perry 1016: static inline struct l2_bucket *
1.134 thorpej 1017: pmap_get_l2_bucket(pmap_t pm, vaddr_t va)
1018: {
1019: struct l2_dtable *l2;
1020: struct l2_bucket *l2b;
1021: u_short l1idx;
1.1 matt 1022:
1.134 thorpej 1023: l1idx = L1_IDX(va);
1.1 matt 1024:
1.134 thorpej 1025: if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL ||
1026: (l2b = &l2->l2_bucket[L2_BUCKET(l1idx)])->l2b_kva == NULL)
1027: return (NULL);
1.1 matt 1028:
1.134 thorpej 1029: return (l2b);
1.1 matt 1030: }
1031:
1032: /*
1.134 thorpej 1033: * Returns a pointer to the L2 bucket associated with the specified pmap
1034: * and VA.
1.1 matt 1035: *
1.134 thorpej 1036: * If no L2 bucket exists, perform the necessary allocations to put an L2
1037: * bucket/page table in place.
1.1 matt 1038: *
1.134 thorpej 1039: * Note that if a new L2 bucket/page was allocated, the caller *must*
1040: * increment the bucket occupancy counter appropriately *before*
1041: * releasing the pmap's lock to ensure no other thread or cpu deallocates
1042: * the bucket/page in the meantime.
1.1 matt 1043: */
1.134 thorpej 1044: static struct l2_bucket *
1045: pmap_alloc_l2_bucket(pmap_t pm, vaddr_t va)
1046: {
1047: struct l2_dtable *l2;
1048: struct l2_bucket *l2b;
1049: u_short l1idx;
1050:
1051: l1idx = L1_IDX(va);
1052:
1053: if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) {
1054: /*
1055: * No mapping at this address, as there is
1056: * no entry in the L1 table.
1057: * Need to allocate a new l2_dtable.
1058: */
1059: if ((l2 = pmap_alloc_l2_dtable()) == NULL)
1060: return (NULL);
1061:
1062: /*
1063: * Link it into the parent pmap
1064: */
1065: pm->pm_l2[L2_IDX(l1idx)] = l2;
1066: }
1.1 matt 1067:
1.134 thorpej 1068: l2b = &l2->l2_bucket[L2_BUCKET(l1idx)];
1.1 matt 1069:
1.10 chris 1070: /*
1.134 thorpej 1071: * Fetch pointer to the L2 page table associated with the address.
1.10 chris 1072: */
1.134 thorpej 1073: if (l2b->l2b_kva == NULL) {
1074: pt_entry_t *ptep;
1075:
1076: /*
1077: * No L2 page table has been allocated. Chances are, this
1078: * is because we just allocated the l2_dtable, above.
1079: */
1080: if ((ptep = pmap_alloc_l2_ptp(&l2b->l2b_phys)) == NULL) {
1081: /*
1082: * Oops, no more L2 page tables available at this
1083: * time. We may need to deallocate the l2_dtable
1084: * if we allocated a new one above.
1085: */
1086: if (l2->l2_occupancy == 0) {
1087: pm->pm_l2[L2_IDX(l1idx)] = NULL;
1088: pmap_free_l2_dtable(l2);
1089: }
1090: return (NULL);
1091: }
1.1 matt 1092:
1.134 thorpej 1093: l2->l2_occupancy++;
1094: l2b->l2b_kva = ptep;
1095: l2b->l2b_l1idx = l1idx;
1096: }
1.16 chris 1097:
1.134 thorpej 1098: return (l2b);
1.1 matt 1099: }
1100:
1101: /*
1.134 thorpej 1102: * One or more mappings in the specified L2 descriptor table have just been
1103: * invalidated.
1.1 matt 1104: *
1.134 thorpej 1105: * Garbage collect the metadata and descriptor table itself if necessary.
1.1 matt 1106: *
1.134 thorpej 1107: * The pmap lock must be acquired when this is called (not necessary
1108: * for the kernel pmap).
1.1 matt 1109: */
1.134 thorpej 1110: static void
1111: pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count)
1.1 matt 1112: {
1.134 thorpej 1113: struct l2_dtable *l2;
1114: pd_entry_t *pl1pd, l1pd;
1115: pt_entry_t *ptep;
1116: u_short l1idx;
1.1 matt 1117:
1.134 thorpej 1118: KDASSERT(count <= l2b->l2b_occupancy);
1.1 matt 1119:
1.134 thorpej 1120: /*
1121: * Update the bucket's reference count according to how many
1122: * PTEs the caller has just invalidated.
1123: */
1124: l2b->l2b_occupancy -= count;
1.1 matt 1125:
1126: /*
1.134 thorpej 1127: * Note:
1128: *
1129: * Level 2 page tables allocated to the kernel pmap are never freed
1130: * as that would require checking all Level 1 page tables and
1131: * removing any references to the Level 2 page table. See also the
1132: * comment elsewhere about never freeing bootstrap L2 descriptors.
1133: *
1134: * We make do with just invalidating the mapping in the L2 table.
1135: *
1136: * This isn't really a big deal in practice and, in fact, leads
1137: * to a performance win over time as we don't need to continually
1138: * alloc/free.
1.1 matt 1139: */
1.134 thorpej 1140: if (l2b->l2b_occupancy > 0 || pm == pmap_kernel())
1141: return;
1.1 matt 1142:
1.134 thorpej 1143: /*
1144: * There are no more valid mappings in this level 2 page table.
1145: * Go ahead and NULL-out the pointer in the bucket, then
1146: * free the page table.
1147: */
1148: l1idx = l2b->l2b_l1idx;
1149: ptep = l2b->l2b_kva;
1150: l2b->l2b_kva = NULL;
1.1 matt 1151:
1.134 thorpej 1152: pl1pd = &pm->pm_l1->l1_kva[l1idx];
1.1 matt 1153:
1.134 thorpej 1154: /*
1155: * If the L1 slot matches the pmap's domain
1156: * number, then invalidate it.
1157: */
1158: l1pd = *pl1pd & (L1_TYPE_MASK | L1_C_DOM_MASK);
1159: if (l1pd == (L1_C_DOM(pm->pm_domain) | L1_TYPE_C)) {
1160: *pl1pd = 0;
1161: PTE_SYNC(pl1pd);
1.1 matt 1162: }
1163:
1.134 thorpej 1164: /*
1165: * Release the L2 descriptor table back to the pool cache.
1166: */
1167: #ifndef PMAP_INCLUDE_PTE_SYNC
1168: pmap_free_l2_ptp(ptep, l2b->l2b_phys);
1169: #else
1170: pmap_free_l2_ptp(!pmap_is_cached(pm), ptep, l2b->l2b_phys);
1171: #endif
1172:
1173: /*
1174: * Update the reference count in the associated l2_dtable
1175: */
1176: l2 = pm->pm_l2[L2_IDX(l1idx)];
1177: if (--l2->l2_occupancy > 0)
1178: return;
1.1 matt 1179:
1.134 thorpej 1180: /*
1181: * There are no more valid mappings in any of the Level 1
1182: * slots managed by this l2_dtable. Go ahead and NULL-out
1183: * the pointer in the parent pmap and free the l2_dtable.
1184: */
1185: pm->pm_l2[L2_IDX(l1idx)] = NULL;
1186: pmap_free_l2_dtable(l2);
1.1 matt 1187: }
1188:
1189: /*
1.134 thorpej 1190: * Pool cache constructors for L2 descriptor tables, metadata and pmap
1191: * structures.
1.1 matt 1192: */
1.134 thorpej 1193: static int
1194: pmap_l2ptp_ctor(void *arg, void *v, int flags)
1.1 matt 1195: {
1.134 thorpej 1196: #ifndef PMAP_INCLUDE_PTE_SYNC
1197: struct l2_bucket *l2b;
1198: pt_entry_t *ptep, pte;
1199: vaddr_t va = (vaddr_t)v & ~PGOFSET;
1200:
1201: /*
1202: * The mappings for these page tables were initially made using
1203: * pmap_kenter_pa() by the pool subsystem. Therefore, the cache-
1204: * mode will not be right for page table mappings. To avoid
1205: * polluting the pmap_kenter_pa() code with a special case for
1206: * page tables, we simply fix up the cache-mode here if it's not
1207: * correct.
1208: */
1209: l2b = pmap_get_l2_bucket(pmap_kernel(), va);
1210: KDASSERT(l2b != NULL);
1211: ptep = &l2b->l2b_kva[l2pte_index(va)];
1212: pte = *ptep;
1.1 matt 1213:
1.134 thorpej 1214: if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) {
1215: /*
1216: * Page tables must have the cache-mode set to Write-Thru.
1217: */
1218: *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt;
1219: PTE_SYNC(ptep);
1220: cpu_tlb_flushD_SE(va);
1221: cpu_cpwait();
1222: }
1223: #endif
1.1 matt 1224:
1.134 thorpej 1225: memset(v, 0, L2_TABLE_SIZE_REAL);
1226: PTE_SYNC_RANGE(v, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t));
1227: return (0);
1.1 matt 1228: }
1229:
1.134 thorpej 1230: static int
1231: pmap_l2dtable_ctor(void *arg, void *v, int flags)
1.93 thorpej 1232: {
1233:
1.134 thorpej 1234: memset(v, 0, sizeof(struct l2_dtable));
1235: return (0);
1236: }
1.93 thorpej 1237:
1.134 thorpej 1238: static int
1239: pmap_pmap_ctor(void *arg, void *v, int flags)
1240: {
1.93 thorpej 1241:
1.134 thorpej 1242: memset(v, 0, sizeof(struct pmap));
1243: return (0);
1.93 thorpej 1244: }
1245:
1.163.4.2! garbled 1246: static void
! 1247: pmap_pinit(pmap_t pm)
! 1248: {
! 1249: struct l2_bucket *l2b;
! 1250:
! 1251: if (vector_page < KERNEL_BASE) {
! 1252: /*
! 1253: * Map the vector page.
! 1254: */
! 1255: pmap_enter(pm, vector_page, systempage.pv_pa,
! 1256: VM_PROT_READ, VM_PROT_READ | PMAP_WIRED);
! 1257: pmap_update(pm);
! 1258:
! 1259: pm->pm_pl1vec = &pm->pm_l1->l1_kva[L1_IDX(vector_page)];
! 1260: l2b = pmap_get_l2_bucket(pm, vector_page);
! 1261: pm->pm_l1vec = l2b->l2b_phys | L1_C_PROTO |
! 1262: L1_C_DOM(pm->pm_domain);
! 1263: } else
! 1264: pm->pm_pl1vec = NULL;
! 1265: }
! 1266:
1.93 thorpej 1267: /*
1.134 thorpej 1268: * Since we have a virtually indexed cache, we may need to inhibit caching if
1269: * there is more than one mapping and at least one of them is writable.
1270: * Since we purge the cache on every context switch, we only need to check for
1271: * other mappings within the same pmap, or kernel_pmap.
1272: * This function is also called when a page is unmapped, to possibly reenable
1273: * caching on any remaining mappings.
1274: *
1275: * The code implements the following logic, where:
1276: *
1277: * KW = # of kernel read/write pages
1278: * KR = # of kernel read only pages
1279: * UW = # of user read/write pages
1280: * UR = # of user read only pages
1281: *
1282: * KC = kernel mapping is cacheable
1283: * UC = user mapping is cacheable
1.93 thorpej 1284: *
1.134 thorpej 1285: * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0
1286: * +---------------------------------------------
1287: * UW=0,UR=0 | --- KC=1 KC=1 KC=0
1288: * UW=0,UR>0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0
1289: * UW=1,UR=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0
1290: * UW>1,UR>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0
1.93 thorpej 1291: */
1.111 thorpej 1292:
1.134 thorpej 1293: static const int pmap_vac_flags[4][4] = {
1294: {-1, 0, 0, PVF_KNC},
1295: {0, 0, PVF_NC, PVF_NC},
1296: {0, PVF_NC, PVF_NC, PVF_NC},
1297: {PVF_UNC, PVF_NC, PVF_NC, PVF_NC}
1298: };
1.93 thorpej 1299:
1.157 perry 1300: static inline int
1.134 thorpej 1301: pmap_get_vac_flags(const struct vm_page *pg)
1302: {
1303: int kidx, uidx;
1.93 thorpej 1304:
1.134 thorpej 1305: kidx = 0;
1306: if (pg->mdpage.kro_mappings || pg->mdpage.krw_mappings > 1)
1307: kidx |= 1;
1308: if (pg->mdpage.krw_mappings)
1309: kidx |= 2;
1310:
1311: uidx = 0;
1312: if (pg->mdpage.uro_mappings || pg->mdpage.urw_mappings > 1)
1313: uidx |= 1;
1314: if (pg->mdpage.urw_mappings)
1315: uidx |= 2;
1.111 thorpej 1316:
1.134 thorpej 1317: return (pmap_vac_flags[uidx][kidx]);
1.111 thorpej 1318: }
1319:
1.157 perry 1320: static inline void
1.134 thorpej 1321: pmap_vac_me_harder(struct vm_page *pg, pmap_t pm, vaddr_t va)
1.111 thorpej 1322: {
1.134 thorpej 1323: int nattr;
1324:
1325: nattr = pmap_get_vac_flags(pg);
1.111 thorpej 1326:
1.134 thorpej 1327: if (nattr < 0) {
1328: pg->mdpage.pvh_attrs &= ~PVF_NC;
1329: return;
1330: }
1.93 thorpej 1331:
1.134 thorpej 1332: if (nattr == 0 && (pg->mdpage.pvh_attrs & PVF_NC) == 0)
1333: return;
1.111 thorpej 1334:
1.134 thorpej 1335: if (pm == pmap_kernel())
1336: pmap_vac_me_kpmap(pg, pm, va);
1337: else
1338: pmap_vac_me_user(pg, pm, va);
1339:
1340: pg->mdpage.pvh_attrs = (pg->mdpage.pvh_attrs & ~PVF_NC) | nattr;
1.93 thorpej 1341: }
1342:
1.134 thorpej 1343: static void
1344: pmap_vac_me_kpmap(struct vm_page *pg, pmap_t pm, vaddr_t va)
1.1 matt 1345: {
1.134 thorpej 1346: u_int u_cacheable, u_entries;
1347: struct pv_entry *pv;
1348: pmap_t last_pmap = pm;
1349:
1350: /*
1351: * Pass one, see if there are both kernel and user pmaps for
1352: * this page. Calculate whether there are user-writable or
1353: * kernel-writable pages.
1354: */
1355: u_cacheable = 0;
1356: for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) {
1357: if (pv->pv_pmap != pm && (pv->pv_flags & PVF_NC) == 0)
1358: u_cacheable++;
1.1 matt 1359: }
1360:
1.134 thorpej 1361: u_entries = pg->mdpage.urw_mappings + pg->mdpage.uro_mappings;
1.1 matt 1362:
1.134 thorpej 1363: /*
1364: * We know we have just been updating a kernel entry, so if
1365: * all user pages are already cacheable, then there is nothing
1366: * further to do.
1367: */
1368: if (pg->mdpage.k_mappings == 0 && u_cacheable == u_entries)
1369: return;
1.1 matt 1370:
1.134 thorpej 1371: if (u_entries) {
1372: /*
1373: * Scan over the list again, for each entry, if it
1374: * might not be set correctly, call pmap_vac_me_user
1375: * to recalculate the settings.
1376: */
1377: for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) {
1378: /*
1379: * We know kernel mappings will get set
1380: * correctly in other calls. We also know
1381: * that if the pmap is the same as last_pmap
1382: * then we've just handled this entry.
1383: */
1384: if (pv->pv_pmap == pm || pv->pv_pmap == last_pmap)
1385: continue;
1.1 matt 1386:
1.134 thorpej 1387: /*
1388: * If there are kernel entries and this page
1389: * is writable but non-cacheable, then we can
1390: * skip this entry also.
1391: */
1392: if (pg->mdpage.k_mappings &&
1393: (pv->pv_flags & (PVF_NC | PVF_WRITE)) ==
1394: (PVF_NC | PVF_WRITE))
1395: continue;
1.111 thorpej 1396:
1.134 thorpej 1397: /*
1398: * Similarly if there are no kernel-writable
1399: * entries and the page is already
1400: * read-only/cacheable.
1401: */
1402: if (pg->mdpage.krw_mappings == 0 &&
1403: (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0)
1404: continue;
1.5 toshii 1405:
1.134 thorpej 1406: /*
1407: * For some of the remaining cases, we know
1408: * that we must recalculate, but for others we
1409: * can't tell if they are correct or not, so
1410: * we recalculate anyway.
1411: */
1412: pmap_vac_me_user(pg, (last_pmap = pv->pv_pmap), 0);
1413: }
1.48 chris 1414:
1.134 thorpej 1415: if (pg->mdpage.k_mappings == 0)
1416: return;
1.111 thorpej 1417: }
1418:
1.134 thorpej 1419: pmap_vac_me_user(pg, pm, va);
1420: }
1.111 thorpej 1421:
1.134 thorpej 1422: static void
1423: pmap_vac_me_user(struct vm_page *pg, pmap_t pm, vaddr_t va)
1424: {
1425: pmap_t kpmap = pmap_kernel();
1426: struct pv_entry *pv, *npv;
1427: struct l2_bucket *l2b;
1428: pt_entry_t *ptep, pte;
1429: u_int entries = 0;
1430: u_int writable = 0;
1431: u_int cacheable_entries = 0;
1432: u_int kern_cacheable = 0;
1433: u_int other_writable = 0;
1.48 chris 1434:
1.134 thorpej 1435: /*
1436: * Count mappings and writable mappings in this pmap.
1437: * Include kernel mappings as part of our own.
1438: * Keep a pointer to the first one.
1439: */
1440: for (pv = npv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) {
1441: /* Count mappings in the same pmap */
1442: if (pm == pv->pv_pmap || kpmap == pv->pv_pmap) {
1443: if (entries++ == 0)
1444: npv = pv;
1.1 matt 1445:
1.134 thorpej 1446: /* Cacheable mappings */
1447: if ((pv->pv_flags & PVF_NC) == 0) {
1448: cacheable_entries++;
1449: if (kpmap == pv->pv_pmap)
1450: kern_cacheable++;
1451: }
1.110 thorpej 1452:
1.134 thorpej 1453: /* Writable mappings */
1454: if (pv->pv_flags & PVF_WRITE)
1455: ++writable;
1456: } else
1457: if (pv->pv_flags & PVF_WRITE)
1458: other_writable = 1;
1459: }
1.1 matt 1460:
1.134 thorpej 1461: /*
1462: * Enable or disable caching as necessary.
1463: * Note: the first entry might be part of the kernel pmap,
1464: * so we can't assume this is indicative of the state of the
1465: * other (maybe non-kpmap) entries.
1466: */
1467: if ((entries > 1 && writable) ||
1468: (entries > 0 && pm == kpmap && other_writable)) {
1469: if (cacheable_entries == 0)
1470: return;
1.1 matt 1471:
1.134 thorpej 1472: for (pv = npv; pv; pv = pv->pv_next) {
1473: if ((pm != pv->pv_pmap && kpmap != pv->pv_pmap) ||
1474: (pv->pv_flags & PVF_NC))
1475: continue;
1.1 matt 1476:
1.134 thorpej 1477: pv->pv_flags |= PVF_NC;
1.26 rearnsha 1478:
1.134 thorpej 1479: l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
1480: ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
1481: pte = *ptep & ~L2_S_CACHE_MASK;
1482:
1483: if ((va != pv->pv_va || pm != pv->pv_pmap) &&
1484: l2pte_valid(pte)) {
1485: if (PV_BEEN_EXECD(pv->pv_flags)) {
1486: pmap_idcache_wbinv_range(pv->pv_pmap,
1487: pv->pv_va, PAGE_SIZE);
1488: pmap_tlb_flushID_SE(pv->pv_pmap,
1489: pv->pv_va);
1490: } else
1491: if (PV_BEEN_REFD(pv->pv_flags)) {
1492: pmap_dcache_wb_range(pv->pv_pmap,
1.160 thorpej 1493: pv->pv_va, PAGE_SIZE, true,
1.134 thorpej 1494: (pv->pv_flags & PVF_WRITE) == 0);
1495: pmap_tlb_flushD_SE(pv->pv_pmap,
1496: pv->pv_va);
1497: }
1498: }
1.1 matt 1499:
1.134 thorpej 1500: *ptep = pte;
1501: PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
1502: }
1503: cpu_cpwait();
1504: } else
1505: if (entries > cacheable_entries) {
1.1 matt 1506: /*
1.134 thorpej 1507: * Turn cacheing back on for some pages. If it is a kernel
1508: * page, only do so if there are no other writable pages.
1.1 matt 1509: */
1.134 thorpej 1510: for (pv = npv; pv; pv = pv->pv_next) {
1511: if (!(pv->pv_flags & PVF_NC) || (pm != pv->pv_pmap &&
1512: (kpmap != pv->pv_pmap || other_writable)))
1513: continue;
1514:
1515: pv->pv_flags &= ~PVF_NC;
1.1 matt 1516:
1.134 thorpej 1517: l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
1518: ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
1519: pte = (*ptep & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode;
1520:
1521: if (l2pte_valid(pte)) {
1522: if (PV_BEEN_EXECD(pv->pv_flags)) {
1523: pmap_tlb_flushID_SE(pv->pv_pmap,
1524: pv->pv_va);
1525: } else
1526: if (PV_BEEN_REFD(pv->pv_flags)) {
1527: pmap_tlb_flushD_SE(pv->pv_pmap,
1528: pv->pv_va);
1529: }
1530: }
1.1 matt 1531:
1.134 thorpej 1532: *ptep = pte;
1533: PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
1534: }
1.111 thorpej 1535: }
1.1 matt 1536: }
1537:
1538: /*
1.134 thorpej 1539: * Modify pte bits for all ptes corresponding to the given physical address.
1540: * We use `maskbits' rather than `clearbits' because we're always passing
1541: * constants and the latter would require an extra inversion at run-time.
1.1 matt 1542: */
1.134 thorpej 1543: static void
1544: pmap_clearbit(struct vm_page *pg, u_int maskbits)
1.1 matt 1545: {
1.134 thorpej 1546: struct l2_bucket *l2b;
1547: struct pv_entry *pv;
1548: pt_entry_t *ptep, npte, opte;
1549: pmap_t pm;
1550: vaddr_t va;
1551: u_int oflags;
1.1 matt 1552:
1.134 thorpej 1553: NPDEBUG(PDB_BITS,
1554: printf("pmap_clearbit: pg %p (0x%08lx) mask 0x%x\n",
1.155 yamt 1555: pg, VM_PAGE_TO_PHYS(pg), maskbits));
1.1 matt 1556:
1.134 thorpej 1557: PMAP_HEAD_TO_MAP_LOCK();
1558: simple_lock(&pg->mdpage.pvh_slock);
1.17 chris 1559:
1560: /*
1.134 thorpej 1561: * Clear saved attributes (modify, reference)
1.17 chris 1562: */
1.134 thorpej 1563: pg->mdpage.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF));
1564:
1565: if (pg->mdpage.pvh_list == NULL) {
1566: simple_unlock(&pg->mdpage.pvh_slock);
1567: PMAP_HEAD_TO_MAP_UNLOCK();
1.17 chris 1568: return;
1.1 matt 1569: }
1570:
1.17 chris 1571: /*
1.134 thorpej 1572: * Loop over all current mappings setting/clearing as appropos
1.17 chris 1573: */
1.134 thorpej 1574: for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) {
1575: va = pv->pv_va;
1576: pm = pv->pv_pmap;
1577: oflags = pv->pv_flags;
1578: pv->pv_flags &= ~maskbits;
1.48 chris 1579:
1.134 thorpej 1580: pmap_acquire_pmap_lock(pm);
1.48 chris 1581:
1.134 thorpej 1582: l2b = pmap_get_l2_bucket(pm, va);
1583: KDASSERT(l2b != NULL);
1.1 matt 1584:
1.134 thorpej 1585: ptep = &l2b->l2b_kva[l2pte_index(va)];
1586: npte = opte = *ptep;
1.114 thorpej 1587:
1.134 thorpej 1588: NPDEBUG(PDB_BITS,
1589: printf(
1590: "pmap_clearbit: pv %p, pm %p, va 0x%08lx, flag 0x%x\n",
1591: pv, pv->pv_pmap, pv->pv_va, oflags));
1.114 thorpej 1592:
1.134 thorpej 1593: if (maskbits & (PVF_WRITE|PVF_MOD)) {
1594: if ((pv->pv_flags & PVF_NC)) {
1595: /*
1596: * Entry is not cacheable:
1597: *
1598: * Don't turn caching on again if this is a
1599: * modified emulation. This would be
1600: * inconsitent with the settings created by
1601: * pmap_vac_me_harder(). Otherwise, it's safe
1602: * to re-enable cacheing.
1603: *
1604: * There's no need to call pmap_vac_me_harder()
1605: * here: all pages are losing their write
1606: * permission.
1607: */
1608: if (maskbits & PVF_WRITE) {
1609: npte |= pte_l2_s_cache_mode;
1610: pv->pv_flags &= ~PVF_NC;
1611: }
1612: } else
1613: if (opte & L2_S_PROT_W) {
1614: /*
1615: * Entry is writable/cacheable: check if pmap
1616: * is current if it is flush it, otherwise it
1617: * won't be in the cache
1618: */
1619: if (PV_BEEN_EXECD(oflags))
1620: pmap_idcache_wbinv_range(pm, pv->pv_va,
1621: PAGE_SIZE);
1622: else
1623: if (PV_BEEN_REFD(oflags))
1624: pmap_dcache_wb_range(pm, pv->pv_va,
1625: PAGE_SIZE,
1.160 thorpej 1626: (maskbits & PVF_REF) ? true : false,
1627: false);
1.134 thorpej 1628: }
1.111 thorpej 1629:
1.134 thorpej 1630: /* make the pte read only */
1631: npte &= ~L2_S_PROT_W;
1.111 thorpej 1632:
1.134 thorpej 1633: if (maskbits & PVF_WRITE) {
1634: /*
1635: * Keep alias accounting up to date
1636: */
1637: if (pv->pv_pmap == pmap_kernel()) {
1638: if (oflags & PVF_WRITE) {
1639: pg->mdpage.krw_mappings--;
1640: pg->mdpage.kro_mappings++;
1641: }
1642: } else
1643: if (oflags & PVF_WRITE) {
1644: pg->mdpage.urw_mappings--;
1645: pg->mdpage.uro_mappings++;
1646: }
1647: }
1648: }
1.1 matt 1649:
1.134 thorpej 1650: if (maskbits & PVF_REF) {
1651: if ((pv->pv_flags & PVF_NC) == 0 &&
1652: (maskbits & (PVF_WRITE|PVF_MOD)) == 0) {
1653: /*
1654: * Check npte here; we may have already
1655: * done the wbinv above, and the validity
1656: * of the PTE is the same for opte and
1657: * npte.
1658: */
1659: if (npte & L2_S_PROT_W) {
1660: if (PV_BEEN_EXECD(oflags))
1661: pmap_idcache_wbinv_range(pm,
1662: pv->pv_va, PAGE_SIZE);
1663: else
1664: if (PV_BEEN_REFD(oflags))
1665: pmap_dcache_wb_range(pm,
1666: pv->pv_va, PAGE_SIZE,
1.160 thorpej 1667: true, false);
1.134 thorpej 1668: } else
1669: if ((npte & L2_TYPE_MASK) != L2_TYPE_INV) {
1670: /* XXXJRT need idcache_inv_range */
1671: if (PV_BEEN_EXECD(oflags))
1672: pmap_idcache_wbinv_range(pm,
1673: pv->pv_va, PAGE_SIZE);
1674: else
1675: if (PV_BEEN_REFD(oflags))
1676: pmap_dcache_wb_range(pm,
1677: pv->pv_va, PAGE_SIZE,
1.160 thorpej 1678: true, true);
1.134 thorpej 1679: }
1680: }
1.1 matt 1681:
1.134 thorpej 1682: /*
1683: * Make the PTE invalid so that we will take a
1684: * page fault the next time the mapping is
1685: * referenced.
1686: */
1687: npte &= ~L2_TYPE_MASK;
1688: npte |= L2_TYPE_INV;
1689: }
1.1 matt 1690:
1.134 thorpej 1691: if (npte != opte) {
1692: *ptep = npte;
1693: PTE_SYNC(ptep);
1694: /* Flush the TLB entry if a current pmap. */
1695: if (PV_BEEN_EXECD(oflags))
1696: pmap_tlb_flushID_SE(pm, pv->pv_va);
1697: else
1698: if (PV_BEEN_REFD(oflags))
1699: pmap_tlb_flushD_SE(pm, pv->pv_va);
1700: }
1.1 matt 1701:
1.134 thorpej 1702: pmap_release_pmap_lock(pm);
1.133 thorpej 1703:
1.134 thorpej 1704: NPDEBUG(PDB_BITS,
1705: printf("pmap_clearbit: pm %p va 0x%lx opte 0x%08x npte 0x%08x\n",
1706: pm, va, opte, npte));
1707: }
1.133 thorpej 1708:
1.134 thorpej 1709: simple_unlock(&pg->mdpage.pvh_slock);
1710: PMAP_HEAD_TO_MAP_UNLOCK();
1.1 matt 1711: }
1712:
1713: /*
1.134 thorpej 1714: * pmap_clean_page()
1715: *
1716: * This is a local function used to work out the best strategy to clean
1717: * a single page referenced by its entry in the PV table. It's used by
1718: * pmap_copy_page, pmap_zero page and maybe some others later on.
1719: *
1720: * Its policy is effectively:
1721: * o If there are no mappings, we don't bother doing anything with the cache.
1722: * o If there is one mapping, we clean just that page.
1723: * o If there are multiple mappings, we clean the entire cache.
1724: *
1725: * So that some functions can be further optimised, it returns 0 if it didn't
1726: * clean the entire cache, or 1 if it did.
1727: *
1728: * XXX One bug in this routine is that if the pv_entry has a single page
1729: * mapped at 0x00000000 a whole cache clean will be performed rather than
1730: * just the 1 page. Since this should not occur in everyday use and if it does
1731: * it will just result in not the most efficient clean for the page.
1.1 matt 1732: */
1.134 thorpej 1733: static int
1.159 thorpej 1734: pmap_clean_page(struct pv_entry *pv, bool is_src)
1.1 matt 1735: {
1.134 thorpej 1736: pmap_t pm, pm_to_clean = NULL;
1737: struct pv_entry *npv;
1738: u_int cache_needs_cleaning = 0;
1739: u_int flags = 0;
1740: vaddr_t page_to_clean = 0;
1.1 matt 1741:
1.134 thorpej 1742: if (pv == NULL) {
1743: /* nothing mapped in so nothing to flush */
1.17 chris 1744: return (0);
1.108 thorpej 1745: }
1.17 chris 1746:
1.108 thorpej 1747: /*
1.134 thorpej 1748: * Since we flush the cache each time we change to a different
1749: * user vmspace, we only need to flush the page if it is in the
1750: * current pmap.
1.17 chris 1751: */
1752: if (curproc)
1.134 thorpej 1753: pm = curproc->p_vmspace->vm_map.pmap;
1.17 chris 1754: else
1.134 thorpej 1755: pm = pmap_kernel();
1.17 chris 1756:
1757: for (npv = pv; npv; npv = npv->pv_next) {
1.134 thorpej 1758: if (npv->pv_pmap == pmap_kernel() || npv->pv_pmap == pm) {
1759: flags |= npv->pv_flags;
1.108 thorpej 1760: /*
1761: * The page is mapped non-cacheable in
1.17 chris 1762: * this map. No need to flush the cache.
1763: */
1.78 thorpej 1764: if (npv->pv_flags & PVF_NC) {
1.17 chris 1765: #ifdef DIAGNOSTIC
1766: if (cache_needs_cleaning)
1767: panic("pmap_clean_page: "
1.108 thorpej 1768: "cache inconsistency");
1.17 chris 1769: #endif
1770: break;
1.108 thorpej 1771: } else if (is_src && (npv->pv_flags & PVF_WRITE) == 0)
1.17 chris 1772: continue;
1.108 thorpej 1773: if (cache_needs_cleaning) {
1.17 chris 1774: page_to_clean = 0;
1775: break;
1.134 thorpej 1776: } else {
1.17 chris 1777: page_to_clean = npv->pv_va;
1.134 thorpej 1778: pm_to_clean = npv->pv_pmap;
1779: }
1780: cache_needs_cleaning = 1;
1.17 chris 1781: }
1.1 matt 1782: }
1783:
1.108 thorpej 1784: if (page_to_clean) {
1.134 thorpej 1785: if (PV_BEEN_EXECD(flags))
1786: pmap_idcache_wbinv_range(pm_to_clean, page_to_clean,
1787: PAGE_SIZE);
1788: else
1789: pmap_dcache_wb_range(pm_to_clean, page_to_clean,
1790: PAGE_SIZE, !is_src, (flags & PVF_WRITE) == 0);
1.108 thorpej 1791: } else if (cache_needs_cleaning) {
1.134 thorpej 1792: if (PV_BEEN_EXECD(flags))
1793: pmap_idcache_wbinv_all(pm);
1794: else
1795: pmap_dcache_wbinv_all(pm);
1.1 matt 1796: return (1);
1797: }
1798: return (0);
1799: }
1800:
1801: /*
1.134 thorpej 1802: * Routine: pmap_page_remove
1803: * Function:
1804: * Removes this physical page from
1805: * all physical maps in which it resides.
1806: * Reflects back modify bits to the pager.
1.1 matt 1807: */
1.134 thorpej 1808: static void
1809: pmap_page_remove(struct vm_page *pg)
1.1 matt 1810: {
1.134 thorpej 1811: struct l2_bucket *l2b;
1812: struct pv_entry *pv, *npv;
1813: pmap_t pm, curpm;
1814: pt_entry_t *ptep, pte;
1.159 thorpej 1815: bool flush;
1.134 thorpej 1816: u_int flags;
1817:
1818: NPDEBUG(PDB_FOLLOW,
1.155 yamt 1819: printf("pmap_page_remove: pg %p (0x%08lx)\n", pg,
1820: VM_PAGE_TO_PHYS(pg)));
1.71 thorpej 1821:
1.134 thorpej 1822: PMAP_HEAD_TO_MAP_LOCK();
1823: simple_lock(&pg->mdpage.pvh_slock);
1.1 matt 1824:
1.134 thorpej 1825: pv = pg->mdpage.pvh_list;
1826: if (pv == NULL) {
1827: simple_unlock(&pg->mdpage.pvh_slock);
1828: PMAP_HEAD_TO_MAP_UNLOCK();
1829: return;
1830: }
1.79 thorpej 1831:
1.1 matt 1832: /*
1.134 thorpej 1833: * Clear alias counts
1.1 matt 1834: */
1.134 thorpej 1835: pg->mdpage.k_mappings = 0;
1836: pg->mdpage.urw_mappings = pg->mdpage.uro_mappings = 0;
1837:
1.160 thorpej 1838: flush = false;
1.134 thorpej 1839: flags = 0;
1840: if (curproc)
1841: curpm = curproc->p_vmspace->vm_map.pmap;
1842: else
1843: curpm = pmap_kernel();
1844:
1.160 thorpej 1845: pmap_clean_page(pv, false);
1.134 thorpej 1846:
1847: while (pv) {
1848: pm = pv->pv_pmap;
1.160 thorpej 1849: if (flush == false && (pm == curpm || pm == pmap_kernel()))
1850: flush = true;
1.134 thorpej 1851:
1852: pmap_acquire_pmap_lock(pm);
1853:
1854: l2b = pmap_get_l2_bucket(pm, pv->pv_va);
1855: KDASSERT(l2b != NULL);
1856:
1857: ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
1858: pte = *ptep;
1859:
1860: /*
1861: * Update statistics
1862: */
1863: --pm->pm_stats.resident_count;
1864:
1865: /* Wired bit */
1866: if (pv->pv_flags & PVF_WIRED)
1867: --pm->pm_stats.wired_count;
1.88 thorpej 1868:
1.134 thorpej 1869: flags |= pv->pv_flags;
1.88 thorpej 1870:
1.134 thorpej 1871: /*
1872: * Invalidate the PTEs.
1873: */
1874: *ptep = 0;
1875: PTE_SYNC_CURRENT(pm, ptep);
1876: pmap_free_l2_bucket(pm, l2b, 1);
1.88 thorpej 1877:
1.134 thorpej 1878: npv = pv->pv_next;
1879: pool_put(&pmap_pv_pool, pv);
1880: pv = npv;
1881: pmap_release_pmap_lock(pm);
1882: }
1883: pg->mdpage.pvh_list = NULL;
1884: simple_unlock(&pg->mdpage.pvh_slock);
1885: PMAP_HEAD_TO_MAP_UNLOCK();
1.88 thorpej 1886:
1.134 thorpej 1887: if (flush) {
1.152 scw 1888: /*
1889: * Note: We can't use pmap_tlb_flush{I,}D() here since that
1890: * would need a subsequent call to pmap_update() to ensure
1891: * curpm->pm_cstate.cs_all is reset. Our callers are not
1892: * required to do that (see pmap(9)), so we can't modify
1893: * the current pmap's state.
1894: */
1.134 thorpej 1895: if (PV_BEEN_EXECD(flags))
1.152 scw 1896: cpu_tlb_flushID();
1.134 thorpej 1897: else
1.152 scw 1898: cpu_tlb_flushD();
1.134 thorpej 1899: }
1.88 thorpej 1900: cpu_cpwait();
1901: }
1.1 matt 1902:
1.134 thorpej 1903: /*
1904: * pmap_t pmap_create(void)
1905: *
1906: * Create a new pmap structure from scratch.
1.17 chris 1907: */
1.134 thorpej 1908: pmap_t
1909: pmap_create(void)
1.17 chris 1910: {
1.134 thorpej 1911: pmap_t pm;
1912:
1913: pm = pool_cache_get(&pmap_pmap_cache, PR_WAITOK);
1.79 thorpej 1914:
1.134 thorpej 1915: simple_lock_init(&pm->pm_lock);
1916: pm->pm_obj.pgops = NULL; /* currently not a mappable object */
1917: TAILQ_INIT(&pm->pm_obj.memq);
1918: pm->pm_obj.uo_npages = 0;
1919: pm->pm_obj.uo_refs = 1;
1920: pm->pm_stats.wired_count = 0;
1921: pm->pm_stats.resident_count = 1;
1922: pm->pm_cstate.cs_all = 0;
1923: pmap_alloc_l1(pm);
1.79 thorpej 1924:
1.17 chris 1925: /*
1.134 thorpej 1926: * Note: The pool cache ensures that the pm_l2[] array is already
1927: * initialised to zero.
1.17 chris 1928: */
1.32 thorpej 1929:
1.134 thorpej 1930: pmap_pinit(pm);
1931:
1932: LIST_INSERT_HEAD(&pmap_pmaps, pm, pm_list);
1.17 chris 1933:
1.134 thorpej 1934: return (pm);
1.17 chris 1935: }
1.134 thorpej 1936:
1.1 matt 1937: /*
1.134 thorpej 1938: * void pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot,
1939: * int flags)
1940: *
1941: * Insert the given physical page (p) at
1942: * the specified virtual address (v) in the
1943: * target physical map with the protection requested.
1.1 matt 1944: *
1.134 thorpej 1945: * NB: This is the only routine which MAY NOT lazy-evaluate
1946: * or lose information. That is, this routine must actually
1947: * insert this page into the given map NOW.
1.1 matt 1948: */
1.134 thorpej 1949: int
1950: pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags)
1.1 matt 1951: {
1.134 thorpej 1952: struct l2_bucket *l2b;
1953: struct vm_page *pg, *opg;
1954: struct pv_entry *pve;
1955: pt_entry_t *ptep, npte, opte;
1956: u_int nflags;
1957: u_int oflags;
1.71 thorpej 1958:
1.134 thorpej 1959: NPDEBUG(PDB_ENTER, printf("pmap_enter: pm %p va 0x%lx pa 0x%lx prot %x flag %x\n", pm, va, pa, prot, flags));
1.71 thorpej 1960:
1.134 thorpej 1961: KDASSERT((flags & PMAP_WIRED) == 0 || (flags & VM_PROT_ALL) != 0);
1962: KDASSERT(((va | pa) & PGOFSET) == 0);
1.79 thorpej 1963:
1.71 thorpej 1964: /*
1.134 thorpej 1965: * Get a pointer to the page. Later on in this function, we
1966: * test for a managed page by checking pg != NULL.
1.71 thorpej 1967: */
1.134 thorpej 1968: pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL;
1969:
1970: nflags = 0;
1971: if (prot & VM_PROT_WRITE)
1972: nflags |= PVF_WRITE;
1973: if (prot & VM_PROT_EXECUTE)
1974: nflags |= PVF_EXEC;
1975: if (flags & PMAP_WIRED)
1976: nflags |= PVF_WIRED;
1977:
1978: PMAP_MAP_TO_HEAD_LOCK();
1979: pmap_acquire_pmap_lock(pm);
1.1 matt 1980:
1981: /*
1.134 thorpej 1982: * Fetch the L2 bucket which maps this page, allocating one if
1983: * necessary for user pmaps.
1.1 matt 1984: */
1.134 thorpej 1985: if (pm == pmap_kernel())
1986: l2b = pmap_get_l2_bucket(pm, va);
1987: else
1988: l2b = pmap_alloc_l2_bucket(pm, va);
1989: if (l2b == NULL) {
1990: if (flags & PMAP_CANFAIL) {
1991: pmap_release_pmap_lock(pm);
1992: PMAP_MAP_TO_HEAD_UNLOCK();
1993: return (ENOMEM);
1994: }
1995: panic("pmap_enter: failed to allocate L2 bucket");
1996: }
1997: ptep = &l2b->l2b_kva[l2pte_index(va)];
1998: opte = *ptep;
1999: npte = pa;
2000: oflags = 0;
1.88 thorpej 2001:
1.134 thorpej 2002: if (opte) {
2003: /*
2004: * There is already a mapping at this address.
2005: * If the physical address is different, lookup the
2006: * vm_page.
2007: */
2008: if (l2pte_pa(opte) != pa)
2009: opg = PHYS_TO_VM_PAGE(l2pte_pa(opte));
2010: else
2011: opg = pg;
2012: } else
2013: opg = NULL;
1.88 thorpej 2014:
1.134 thorpej 2015: if (pg) {
2016: /*
2017: * This is to be a managed mapping.
2018: */
2019: if ((flags & VM_PROT_ALL) ||
2020: (pg->mdpage.pvh_attrs & PVF_REF)) {
2021: /*
2022: * - The access type indicates that we don't need
2023: * to do referenced emulation.
2024: * OR
2025: * - The physical page has already been referenced
2026: * so no need to re-do referenced emulation here.
2027: */
2028: npte |= L2_S_PROTO;
1.88 thorpej 2029:
1.134 thorpej 2030: nflags |= PVF_REF;
1.88 thorpej 2031:
1.134 thorpej 2032: if ((prot & VM_PROT_WRITE) != 0 &&
2033: ((flags & VM_PROT_WRITE) != 0 ||
2034: (pg->mdpage.pvh_attrs & PVF_MOD) != 0)) {
2035: /*
2036: * This is a writable mapping, and the
2037: * page's mod state indicates it has
2038: * already been modified. Make it
2039: * writable from the outset.
2040: */
2041: npte |= L2_S_PROT_W;
2042: nflags |= PVF_MOD;
2043: }
2044: } else {
2045: /*
2046: * Need to do page referenced emulation.
2047: */
2048: npte |= L2_TYPE_INV;
2049: }
1.88 thorpej 2050:
1.134 thorpej 2051: npte |= pte_l2_s_cache_mode;
1.1 matt 2052:
1.134 thorpej 2053: if (pg == opg) {
2054: /*
2055: * We're changing the attrs of an existing mapping.
2056: */
2057: simple_lock(&pg->mdpage.pvh_slock);
2058: oflags = pmap_modify_pv(pg, pm, va,
2059: PVF_WRITE | PVF_EXEC | PVF_WIRED |
2060: PVF_MOD | PVF_REF, nflags);
2061: simple_unlock(&pg->mdpage.pvh_slock);
1.1 matt 2062:
1.134 thorpej 2063: /*
2064: * We may need to flush the cache if we're
2065: * doing rw-ro...
2066: */
2067: if (pm->pm_cstate.cs_cache_d &&
2068: (oflags & PVF_NC) == 0 &&
2069: (opte & L2_S_PROT_W) != 0 &&
2070: (prot & VM_PROT_WRITE) == 0)
2071: cpu_dcache_wb_range(va, PAGE_SIZE);
2072: } else {
2073: /*
2074: * New mapping, or changing the backing page
2075: * of an existing mapping.
2076: */
2077: if (opg) {
2078: /*
2079: * Replacing an existing mapping with a new one.
2080: * It is part of our managed memory so we
2081: * must remove it from the PV list
2082: */
2083: simple_lock(&opg->mdpage.pvh_slock);
1.156 scw 2084: pve = pmap_remove_pv(opg, pm, va, 0);
1.134 thorpej 2085: pmap_vac_me_harder(opg, pm, 0);
2086: simple_unlock(&opg->mdpage.pvh_slock);
2087: oflags = pve->pv_flags;
1.1 matt 2088:
1.134 thorpej 2089: /*
2090: * If the old mapping was valid (ref/mod
2091: * emulation creates 'invalid' mappings
2092: * initially) then make sure to frob
2093: * the cache.
2094: */
2095: if ((oflags & PVF_NC) == 0 &&
2096: l2pte_valid(opte)) {
2097: if (PV_BEEN_EXECD(oflags)) {
2098: pmap_idcache_wbinv_range(pm, va,
2099: PAGE_SIZE);
2100: } else
2101: if (PV_BEEN_REFD(oflags)) {
2102: pmap_dcache_wb_range(pm, va,
1.160 thorpej 2103: PAGE_SIZE, true,
1.134 thorpej 2104: (oflags & PVF_WRITE) == 0);
2105: }
2106: }
2107: } else
2108: if ((pve = pool_get(&pmap_pv_pool, PR_NOWAIT)) == NULL){
2109: if ((flags & PMAP_CANFAIL) == 0)
2110: panic("pmap_enter: no pv entries");
2111:
2112: if (pm != pmap_kernel())
2113: pmap_free_l2_bucket(pm, l2b, 0);
2114: pmap_release_pmap_lock(pm);
2115: PMAP_MAP_TO_HEAD_UNLOCK();
2116: NPDEBUG(PDB_ENTER,
2117: printf("pmap_enter: ENOMEM\n"));
2118: return (ENOMEM);
2119: }
1.25 rearnsha 2120:
1.134 thorpej 2121: pmap_enter_pv(pg, pve, pm, va, nflags);
1.25 rearnsha 2122: }
1.134 thorpej 2123: } else {
2124: /*
2125: * We're mapping an unmanaged page.
2126: * These are always readable, and possibly writable, from
2127: * the get go as we don't need to track ref/mod status.
2128: */
2129: npte |= L2_S_PROTO;
2130: if (prot & VM_PROT_WRITE)
2131: npte |= L2_S_PROT_W;
1.25 rearnsha 2132:
1.134 thorpej 2133: /*
2134: * Make sure the vector table is mapped cacheable
2135: */
2136: if (pm != pmap_kernel() && va == vector_page)
2137: npte |= pte_l2_s_cache_mode;
1.25 rearnsha 2138:
1.134 thorpej 2139: if (opg) {
2140: /*
2141: * Looks like there's an existing 'managed' mapping
2142: * at this address.
1.25 rearnsha 2143: */
1.134 thorpej 2144: simple_lock(&opg->mdpage.pvh_slock);
1.156 scw 2145: pve = pmap_remove_pv(opg, pm, va, 0);
1.134 thorpej 2146: pmap_vac_me_harder(opg, pm, 0);
2147: simple_unlock(&opg->mdpage.pvh_slock);
2148: oflags = pve->pv_flags;
2149:
2150: if ((oflags & PVF_NC) == 0 && l2pte_valid(opte)) {
2151: if (PV_BEEN_EXECD(oflags))
2152: pmap_idcache_wbinv_range(pm, va,
2153: PAGE_SIZE);
2154: else
2155: if (PV_BEEN_REFD(oflags))
2156: pmap_dcache_wb_range(pm, va, PAGE_SIZE,
1.160 thorpej 2157: true, (oflags & PVF_WRITE) == 0);
1.134 thorpej 2158: }
2159: pool_put(&pmap_pv_pool, pve);
1.25 rearnsha 2160: }
2161: }
2162:
1.134 thorpej 2163: /*
2164: * Make sure userland mappings get the right permissions
2165: */
2166: if (pm != pmap_kernel() && va != vector_page)
2167: npte |= L2_S_PROT_U;
1.25 rearnsha 2168:
1.134 thorpej 2169: /*
2170: * Keep the stats up to date
2171: */
2172: if (opte == 0) {
2173: l2b->l2b_occupancy++;
2174: pm->pm_stats.resident_count++;
2175: }
1.1 matt 2176:
1.134 thorpej 2177: NPDEBUG(PDB_ENTER,
2178: printf("pmap_enter: opte 0x%08x npte 0x%08x\n", opte, npte));
1.1 matt 2179:
2180: /*
1.134 thorpej 2181: * If this is just a wiring change, the two PTEs will be
2182: * identical, so there's no need to update the page table.
1.1 matt 2183: */
1.134 thorpej 2184: if (npte != opte) {
1.159 thorpej 2185: bool is_cached = pmap_is_cached(pm);
1.1 matt 2186:
1.134 thorpej 2187: *ptep = npte;
2188: if (is_cached) {
2189: /*
2190: * We only need to frob the cache/tlb if this pmap
2191: * is current
2192: */
2193: PTE_SYNC(ptep);
2194: if (va != vector_page && l2pte_valid(npte)) {
1.25 rearnsha 2195: /*
1.134 thorpej 2196: * This mapping is likely to be accessed as
2197: * soon as we return to userland. Fix up the
2198: * L1 entry to avoid taking another
2199: * page/domain fault.
1.25 rearnsha 2200: */
1.134 thorpej 2201: pd_entry_t *pl1pd, l1pd;
2202:
2203: pl1pd = &pm->pm_l1->l1_kva[L1_IDX(va)];
2204: l1pd = l2b->l2b_phys | L1_C_DOM(pm->pm_domain) |
2205: L1_C_PROTO;
2206: if (*pl1pd != l1pd) {
2207: *pl1pd = l1pd;
2208: PTE_SYNC(pl1pd);
1.12 chris 2209: }
1.1 matt 2210: }
2211: }
1.134 thorpej 2212:
2213: if (PV_BEEN_EXECD(oflags))
2214: pmap_tlb_flushID_SE(pm, va);
2215: else
2216: if (PV_BEEN_REFD(oflags))
2217: pmap_tlb_flushD_SE(pm, va);
2218:
2219: NPDEBUG(PDB_ENTER,
2220: printf("pmap_enter: is_cached %d cs 0x%08x\n",
2221: is_cached, pm->pm_cstate.cs_all));
2222:
2223: if (pg != NULL) {
2224: simple_lock(&pg->mdpage.pvh_slock);
2225: pmap_vac_me_harder(pg, pm, va);
2226: simple_unlock(&pg->mdpage.pvh_slock);
1.1 matt 2227: }
2228: }
1.134 thorpej 2229:
2230: pmap_release_pmap_lock(pm);
2231: PMAP_MAP_TO_HEAD_UNLOCK();
2232:
2233: return (0);
1.1 matt 2234: }
2235:
2236: /*
2237: * pmap_remove()
2238: *
2239: * pmap_remove is responsible for nuking a number of mappings for a range
2240: * of virtual address space in the current pmap. To do this efficiently
2241: * is interesting, because in a number of cases a wide virtual address
2242: * range may be supplied that contains few actual mappings. So, the
2243: * optimisations are:
1.134 thorpej 2244: * 1. Skip over hunks of address space for which no L1 or L2 entry exists.
1.1 matt 2245: * 2. Build up a list of pages we've hit, up to a maximum, so we can
2246: * maybe do just a partial cache clean. This path of execution is
2247: * complicated by the fact that the cache must be flushed _before_
2248: * the PTE is nuked, being a VAC :-)
1.134 thorpej 2249: * 3. If we're called after UVM calls pmap_remove_all(), we can defer
2250: * all invalidations until pmap_update(), since pmap_remove_all() has
2251: * already flushed the cache.
2252: * 4. Maybe later fast-case a single page, but I don't think this is
1.1 matt 2253: * going to make _that_ much difference overall.
2254: */
2255:
1.134 thorpej 2256: #define PMAP_REMOVE_CLEAN_LIST_SIZE 3
1.1 matt 2257:
2258: void
1.156 scw 2259: pmap_do_remove(pmap_t pm, vaddr_t sva, vaddr_t eva, int skip_wired)
1.1 matt 2260: {
1.134 thorpej 2261: struct l2_bucket *l2b;
2262: vaddr_t next_bucket;
2263: pt_entry_t *ptep;
2264: u_int cleanlist_idx, total, cnt;
2265: struct {
1.1 matt 2266: vaddr_t va;
2267: pt_entry_t *pte;
2268: } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE];
1.134 thorpej 2269: u_int mappings, is_exec, is_refd;
1.1 matt 2270:
1.156 scw 2271: NPDEBUG(PDB_REMOVE, printf("pmap_do_remove: pmap=%p sva=%08lx "
2272: "eva=%08lx\n", pm, sva, eva));
1.1 matt 2273:
1.17 chris 2274: /*
1.134 thorpej 2275: * we lock in the pmap => pv_head direction
1.17 chris 2276: */
2277: PMAP_MAP_TO_HEAD_LOCK();
1.134 thorpej 2278: pmap_acquire_pmap_lock(pm);
2279:
2280: if (pm->pm_remove_all || !pmap_is_cached(pm)) {
2281: cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1;
2282: if (pm->pm_cstate.cs_tlb == 0)
1.160 thorpej 2283: pm->pm_remove_all = true;
1.134 thorpej 2284: } else
2285: cleanlist_idx = 0;
2286:
2287: total = 0;
2288:
1.1 matt 2289: while (sva < eva) {
1.134 thorpej 2290: /*
2291: * Do one L2 bucket's worth at a time.
2292: */
2293: next_bucket = L2_NEXT_BUCKET(sva);
2294: if (next_bucket > eva)
2295: next_bucket = eva;
2296:
2297: l2b = pmap_get_l2_bucket(pm, sva);
2298: if (l2b == NULL) {
2299: sva = next_bucket;
2300: continue;
2301: }
2302:
2303: ptep = &l2b->l2b_kva[l2pte_index(sva)];
2304:
1.156 scw 2305: for (mappings = 0; sva < next_bucket; sva += PAGE_SIZE, ptep++){
1.134 thorpej 2306: struct vm_page *pg;
2307: pt_entry_t pte;
2308: paddr_t pa;
2309:
2310: pte = *ptep;
1.1 matt 2311:
1.134 thorpej 2312: if (pte == 0) {
1.156 scw 2313: /* Nothing here, move along */
1.1 matt 2314: continue;
2315: }
2316:
1.134 thorpej 2317: pa = l2pte_pa(pte);
2318: is_exec = 0;
2319: is_refd = 1;
1.1 matt 2320:
2321: /*
1.134 thorpej 2322: * Update flags. In a number of circumstances,
2323: * we could cluster a lot of these and do a
2324: * number of sequential pages in one go.
1.1 matt 2325: */
1.134 thorpej 2326: if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
2327: struct pv_entry *pve;
2328: simple_lock(&pg->mdpage.pvh_slock);
1.156 scw 2329: pve = pmap_remove_pv(pg, pm, sva, skip_wired);
1.134 thorpej 2330: pmap_vac_me_harder(pg, pm, 0);
2331: simple_unlock(&pg->mdpage.pvh_slock);
2332: if (pve != NULL) {
1.160 thorpej 2333: if (pm->pm_remove_all == false) {
1.134 thorpej 2334: is_exec =
2335: PV_BEEN_EXECD(pve->pv_flags);
2336: is_refd =
2337: PV_BEEN_REFD(pve->pv_flags);
2338: }
2339: pool_put(&pmap_pv_pool, pve);
1.156 scw 2340: } else
2341: if (skip_wired) {
2342: /* The mapping is wired. Skip it */
2343: continue;
1.134 thorpej 2344: }
1.156 scw 2345: } else
2346: if (skip_wired) {
2347: /* Unmanaged pages are always wired. */
2348: continue;
1.134 thorpej 2349: }
2350:
1.156 scw 2351: mappings++;
2352:
1.134 thorpej 2353: if (!l2pte_valid(pte)) {
1.156 scw 2354: /*
2355: * Ref/Mod emulation is still active for this
2356: * mapping, therefore it is has not yet been
2357: * accessed. No need to frob the cache/tlb.
2358: */
1.134 thorpej 2359: *ptep = 0;
2360: PTE_SYNC_CURRENT(pm, ptep);
2361: continue;
2362: }
1.1 matt 2363:
2364: if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) {
2365: /* Add to the clean list. */
1.134 thorpej 2366: cleanlist[cleanlist_idx].pte = ptep;
2367: cleanlist[cleanlist_idx].va =
2368: sva | (is_exec & 1);
1.1 matt 2369: cleanlist_idx++;
1.134 thorpej 2370: } else
2371: if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) {
1.1 matt 2372: /* Nuke everything if needed. */
1.134 thorpej 2373: pmap_idcache_wbinv_all(pm);
2374: pmap_tlb_flushID(pm);
1.1 matt 2375:
2376: /*
2377: * Roll back the previous PTE list,
2378: * and zero out the current PTE.
2379: */
1.113 thorpej 2380: for (cnt = 0;
1.134 thorpej 2381: cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) {
1.1 matt 2382: *cleanlist[cnt].pte = 0;
2383: }
1.134 thorpej 2384: *ptep = 0;
2385: PTE_SYNC(ptep);
1.1 matt 2386: cleanlist_idx++;
1.160 thorpej 2387: pm->pm_remove_all = true;
1.1 matt 2388: } else {
1.134 thorpej 2389: *ptep = 0;
2390: PTE_SYNC(ptep);
1.160 thorpej 2391: if (pm->pm_remove_all == false) {
1.134 thorpej 2392: if (is_exec)
2393: pmap_tlb_flushID_SE(pm, sva);
2394: else
2395: if (is_refd)
2396: pmap_tlb_flushD_SE(pm, sva);
2397: }
2398: }
2399: }
2400:
2401: /*
2402: * Deal with any left overs
2403: */
2404: if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) {
2405: total += cleanlist_idx;
2406: for (cnt = 0; cnt < cleanlist_idx; cnt++) {
2407: if (pm->pm_cstate.cs_all != 0) {
2408: vaddr_t clva = cleanlist[cnt].va & ~1;
2409: if (cleanlist[cnt].va & 1) {
2410: pmap_idcache_wbinv_range(pm,
2411: clva, PAGE_SIZE);
2412: pmap_tlb_flushID_SE(pm, clva);
2413: } else {
2414: pmap_dcache_wb_range(pm,
1.160 thorpej 2415: clva, PAGE_SIZE, true,
2416: false);
1.134 thorpej 2417: pmap_tlb_flushD_SE(pm, clva);
2418: }
2419: }
2420: *cleanlist[cnt].pte = 0;
2421: PTE_SYNC_CURRENT(pm, cleanlist[cnt].pte);
1.1 matt 2422: }
2423:
2424: /*
1.134 thorpej 2425: * If it looks like we're removing a whole bunch
2426: * of mappings, it's faster to just write-back
2427: * the whole cache now and defer TLB flushes until
2428: * pmap_update() is called.
1.1 matt 2429: */
1.134 thorpej 2430: if (total <= PMAP_REMOVE_CLEAN_LIST_SIZE)
2431: cleanlist_idx = 0;
2432: else {
2433: cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1;
2434: pmap_idcache_wbinv_all(pm);
1.160 thorpej 2435: pm->pm_remove_all = true;
1.134 thorpej 2436: }
2437: }
2438:
2439: pmap_free_l2_bucket(pm, l2b, mappings);
1.156 scw 2440: pm->pm_stats.resident_count -= mappings;
1.134 thorpej 2441: }
2442:
2443: pmap_release_pmap_lock(pm);
2444: PMAP_MAP_TO_HEAD_UNLOCK();
2445: }
2446:
2447: /*
2448: * pmap_kenter_pa: enter an unmanaged, wired kernel mapping
2449: *
2450: * We assume there is already sufficient KVM space available
2451: * to do this, as we can't allocate L2 descriptor tables/metadata
2452: * from here.
2453: */
2454: void
2455: pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot)
2456: {
2457: struct l2_bucket *l2b;
2458: pt_entry_t *ptep, opte;
2459:
2460: NPDEBUG(PDB_KENTER,
2461: printf("pmap_kenter_pa: va 0x%08lx, pa 0x%08lx, prot 0x%x\n",
2462: va, pa, prot));
2463:
2464: l2b = pmap_get_l2_bucket(pmap_kernel(), va);
2465: KDASSERT(l2b != NULL);
2466:
2467: ptep = &l2b->l2b_kva[l2pte_index(va)];
2468: opte = *ptep;
2469:
2470: if (l2pte_valid(opte)) {
2471: cpu_dcache_wbinv_range(va, PAGE_SIZE);
2472: cpu_tlb_flushD_SE(va);
2473: cpu_cpwait();
2474: } else
2475: if (opte == 0)
2476: l2b->l2b_occupancy++;
2477:
2478: *ptep = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) |
2479: pte_l2_s_cache_mode;
2480: PTE_SYNC(ptep);
2481: }
2482:
2483: void
2484: pmap_kremove(vaddr_t va, vsize_t len)
2485: {
2486: struct l2_bucket *l2b;
2487: pt_entry_t *ptep, *sptep, opte;
2488: vaddr_t next_bucket, eva;
2489: u_int mappings;
2490:
2491: NPDEBUG(PDB_KREMOVE, printf("pmap_kremove: va 0x%08lx, len 0x%08lx\n",
2492: va, len));
2493:
2494: eva = va + len;
2495:
2496: while (va < eva) {
2497: next_bucket = L2_NEXT_BUCKET(va);
2498: if (next_bucket > eva)
2499: next_bucket = eva;
2500:
2501: l2b = pmap_get_l2_bucket(pmap_kernel(), va);
2502: KDASSERT(l2b != NULL);
2503:
2504: sptep = ptep = &l2b->l2b_kva[l2pte_index(va)];
2505: mappings = 0;
2506:
2507: while (va < next_bucket) {
2508: opte = *ptep;
2509: if (l2pte_valid(opte)) {
2510: cpu_dcache_wbinv_range(va, PAGE_SIZE);
2511: cpu_tlb_flushD_SE(va);
2512: }
2513: if (opte) {
2514: *ptep = 0;
2515: mappings++;
2516: }
2517: va += PAGE_SIZE;
2518: ptep++;
2519: }
2520: KDASSERT(mappings <= l2b->l2b_occupancy);
2521: l2b->l2b_occupancy -= mappings;
2522: PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep));
2523: }
2524: cpu_cpwait();
2525: }
2526:
1.159 thorpej 2527: bool
1.134 thorpej 2528: pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap)
2529: {
2530: struct l2_dtable *l2;
2531: pd_entry_t *pl1pd, l1pd;
2532: pt_entry_t *ptep, pte;
2533: paddr_t pa;
2534: u_int l1idx;
2535:
2536: pmap_acquire_pmap_lock(pm);
2537:
2538: l1idx = L1_IDX(va);
2539: pl1pd = &pm->pm_l1->l1_kva[l1idx];
2540: l1pd = *pl1pd;
2541:
2542: if (l1pte_section_p(l1pd)) {
2543: /*
2544: * These should only happen for pmap_kernel()
2545: */
2546: KDASSERT(pm == pmap_kernel());
2547: pmap_release_pmap_lock(pm);
2548: pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET);
2549: } else {
2550: /*
2551: * Note that we can't rely on the validity of the L1
2552: * descriptor as an indication that a mapping exists.
2553: * We have to look it up in the L2 dtable.
2554: */
2555: l2 = pm->pm_l2[L2_IDX(l1idx)];
2556:
2557: if (l2 == NULL ||
2558: (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) {
2559: pmap_release_pmap_lock(pm);
1.160 thorpej 2560: return (false);
1.134 thorpej 2561: }
2562:
2563: ptep = &ptep[l2pte_index(va)];
2564: pte = *ptep;
2565: pmap_release_pmap_lock(pm);
2566:
2567: if (pte == 0)
1.160 thorpej 2568: return (false);
1.134 thorpej 2569:
2570: switch (pte & L2_TYPE_MASK) {
2571: case L2_TYPE_L:
2572: pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET);
2573: break;
2574:
2575: default:
2576: pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET);
2577: break;
2578: }
2579: }
2580:
2581: if (pap != NULL)
2582: *pap = pa;
2583:
1.160 thorpej 2584: return (true);
1.134 thorpej 2585: }
2586:
2587: void
2588: pmap_protect(pmap_t pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
2589: {
2590: struct l2_bucket *l2b;
2591: pt_entry_t *ptep, pte;
2592: vaddr_t next_bucket;
2593: u_int flags;
2594: int flush;
2595:
2596: NPDEBUG(PDB_PROTECT,
2597: printf("pmap_protect: pm %p sva 0x%lx eva 0x%lx prot 0x%x\n",
2598: pm, sva, eva, prot));
2599:
2600: if ((prot & VM_PROT_READ) == 0) {
2601: pmap_remove(pm, sva, eva);
2602: return;
2603: }
2604:
2605: if (prot & VM_PROT_WRITE) {
2606: /*
2607: * If this is a read->write transition, just ignore it and let
2608: * uvm_fault() take care of it later.
2609: */
2610: return;
2611: }
2612:
2613: PMAP_MAP_TO_HEAD_LOCK();
2614: pmap_acquire_pmap_lock(pm);
2615:
2616: /*
2617: * OK, at this point, we know we're doing write-protect operation.
2618: * If the pmap is active, write-back the range.
2619: */
1.160 thorpej 2620: pmap_dcache_wb_range(pm, sva, eva - sva, false, false);
1.134 thorpej 2621:
2622: flush = ((eva - sva) >= (PAGE_SIZE * 4)) ? 0 : -1;
2623: flags = 0;
2624:
2625: while (sva < eva) {
2626: next_bucket = L2_NEXT_BUCKET(sva);
2627: if (next_bucket > eva)
2628: next_bucket = eva;
2629:
2630: l2b = pmap_get_l2_bucket(pm, sva);
2631: if (l2b == NULL) {
2632: sva = next_bucket;
2633: continue;
2634: }
2635:
2636: ptep = &l2b->l2b_kva[l2pte_index(sva)];
2637:
2638: while (sva < next_bucket) {
2639: if ((pte = *ptep) != 0 && (pte & L2_S_PROT_W) != 0) {
2640: struct vm_page *pg;
2641: u_int f;
2642:
2643: pg = PHYS_TO_VM_PAGE(l2pte_pa(pte));
2644: pte &= ~L2_S_PROT_W;
2645: *ptep = pte;
2646: PTE_SYNC(ptep);
2647:
2648: if (pg != NULL) {
2649: simple_lock(&pg->mdpage.pvh_slock);
2650: f = pmap_modify_pv(pg, pm, sva,
2651: PVF_WRITE, 0);
2652: pmap_vac_me_harder(pg, pm, sva);
2653: simple_unlock(&pg->mdpage.pvh_slock);
2654: } else
2655: f = PVF_REF | PVF_EXEC;
2656:
2657: if (flush >= 0) {
2658: flush++;
2659: flags |= f;
2660: } else
2661: if (PV_BEEN_EXECD(f))
2662: pmap_tlb_flushID_SE(pm, sva);
2663: else
2664: if (PV_BEEN_REFD(f))
2665: pmap_tlb_flushD_SE(pm, sva);
1.1 matt 2666: }
1.134 thorpej 2667:
2668: sva += PAGE_SIZE;
2669: ptep++;
2670: }
1.1 matt 2671: }
2672:
1.134 thorpej 2673: pmap_release_pmap_lock(pm);
2674: PMAP_MAP_TO_HEAD_UNLOCK();
2675:
2676: if (flush) {
2677: if (PV_BEEN_EXECD(flags))
2678: pmap_tlb_flushID(pm);
2679: else
2680: if (PV_BEEN_REFD(flags))
2681: pmap_tlb_flushD(pm);
2682: }
2683: }
2684:
2685: void
2686: pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
2687: {
2688:
2689: NPDEBUG(PDB_PROTECT,
2690: printf("pmap_page_protect: pg %p (0x%08lx), prot 0x%x\n",
1.155 yamt 2691: pg, VM_PAGE_TO_PHYS(pg), prot));
1.134 thorpej 2692:
2693: switch(prot) {
2694: case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE:
2695: case VM_PROT_READ|VM_PROT_WRITE:
2696: return;
2697:
2698: case VM_PROT_READ:
2699: case VM_PROT_READ|VM_PROT_EXECUTE:
2700: pmap_clearbit(pg, PVF_WRITE);
2701: break;
2702:
2703: default:
2704: pmap_page_remove(pg);
2705: break;
2706: }
2707: }
2708:
2709: /*
2710: * pmap_clear_modify:
2711: *
2712: * Clear the "modified" attribute for a page.
2713: */
1.159 thorpej 2714: bool
1.134 thorpej 2715: pmap_clear_modify(struct vm_page *pg)
2716: {
1.159 thorpej 2717: bool rv;
1.134 thorpej 2718:
2719: if (pg->mdpage.pvh_attrs & PVF_MOD) {
1.160 thorpej 2720: rv = true;
1.134 thorpej 2721: pmap_clearbit(pg, PVF_MOD);
2722: } else
1.160 thorpej 2723: rv = false;
1.134 thorpej 2724:
2725: return (rv);
2726: }
2727:
2728: /*
2729: * pmap_clear_reference:
2730: *
2731: * Clear the "referenced" attribute for a page.
2732: */
1.159 thorpej 2733: bool
1.134 thorpej 2734: pmap_clear_reference(struct vm_page *pg)
2735: {
1.159 thorpej 2736: bool rv;
1.134 thorpej 2737:
2738: if (pg->mdpage.pvh_attrs & PVF_REF) {
1.160 thorpej 2739: rv = true;
1.134 thorpej 2740: pmap_clearbit(pg, PVF_REF);
2741: } else
1.160 thorpej 2742: rv = false;
1.134 thorpej 2743:
2744: return (rv);
2745: }
2746:
2747: /*
2748: * pmap_is_modified:
2749: *
2750: * Test if a page has the "modified" attribute.
2751: */
2752: /* See <arm/arm32/pmap.h> */
2753:
2754: /*
2755: * pmap_is_referenced:
2756: *
2757: * Test if a page has the "referenced" attribute.
2758: */
2759: /* See <arm/arm32/pmap.h> */
2760:
2761: int
2762: pmap_fault_fixup(pmap_t pm, vaddr_t va, vm_prot_t ftype, int user)
2763: {
2764: struct l2_dtable *l2;
2765: struct l2_bucket *l2b;
2766: pd_entry_t *pl1pd, l1pd;
2767: pt_entry_t *ptep, pte;
2768: paddr_t pa;
2769: u_int l1idx;
2770: int rv = 0;
2771:
2772: PMAP_MAP_TO_HEAD_LOCK();
2773: pmap_acquire_pmap_lock(pm);
2774:
2775: l1idx = L1_IDX(va);
2776:
2777: /*
2778: * If there is no l2_dtable for this address, then the process
2779: * has no business accessing it.
2780: *
2781: * Note: This will catch userland processes trying to access
2782: * kernel addresses.
2783: */
2784: l2 = pm->pm_l2[L2_IDX(l1idx)];
2785: if (l2 == NULL)
2786: goto out;
2787:
1.1 matt 2788: /*
1.134 thorpej 2789: * Likewise if there is no L2 descriptor table
1.1 matt 2790: */
1.134 thorpej 2791: l2b = &l2->l2_bucket[L2_BUCKET(l1idx)];
2792: if (l2b->l2b_kva == NULL)
2793: goto out;
2794:
2795: /*
2796: * Check the PTE itself.
2797: */
2798: ptep = &l2b->l2b_kva[l2pte_index(va)];
2799: pte = *ptep;
2800: if (pte == 0)
2801: goto out;
2802:
2803: /*
2804: * Catch a userland access to the vector page mapped at 0x0
2805: */
2806: if (user && (pte & L2_S_PROT_U) == 0)
2807: goto out;
2808:
2809: pa = l2pte_pa(pte);
2810:
2811: if ((ftype & VM_PROT_WRITE) && (pte & L2_S_PROT_W) == 0) {
2812: /*
2813: * This looks like a good candidate for "page modified"
2814: * emulation...
2815: */
2816: struct pv_entry *pv;
2817: struct vm_page *pg;
2818:
2819: /* Extract the physical address of the page */
2820: if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL)
2821: goto out;
2822:
2823: /* Get the current flags for this page. */
2824: simple_lock(&pg->mdpage.pvh_slock);
2825:
2826: pv = pmap_find_pv(pg, pm, va);
2827: if (pv == NULL) {
2828: simple_unlock(&pg->mdpage.pvh_slock);
2829: goto out;
2830: }
2831:
2832: /*
2833: * Do the flags say this page is writable? If not then it
2834: * is a genuine write fault. If yes then the write fault is
2835: * our fault as we did not reflect the write access in the
2836: * PTE. Now we know a write has occurred we can correct this
2837: * and also set the modified bit
2838: */
2839: if ((pv->pv_flags & PVF_WRITE) == 0) {
2840: simple_unlock(&pg->mdpage.pvh_slock);
2841: goto out;
2842: }
2843:
2844: NPDEBUG(PDB_FOLLOW,
2845: printf("pmap_fault_fixup: mod emul. pm %p, va 0x%08lx, pa 0x%08lx\n",
1.155 yamt 2846: pm, va, VM_PAGE_TO_PHYS(pg)));
1.134 thorpej 2847:
2848: pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD;
2849: pv->pv_flags |= PVF_REF | PVF_MOD;
2850: simple_unlock(&pg->mdpage.pvh_slock);
2851:
2852: /*
2853: * Re-enable write permissions for the page. No need to call
2854: * pmap_vac_me_harder(), since this is just a
2855: * modified-emulation fault, and the PVF_WRITE bit isn't
2856: * changing. We've already set the cacheable bits based on
2857: * the assumption that we can write to this page.
2858: */
2859: *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W;
2860: PTE_SYNC(ptep);
2861: rv = 1;
2862: } else
2863: if ((pte & L2_TYPE_MASK) == L2_TYPE_INV) {
2864: /*
2865: * This looks like a good candidate for "page referenced"
2866: * emulation.
2867: */
2868: struct pv_entry *pv;
2869: struct vm_page *pg;
2870:
2871: /* Extract the physical address of the page */
2872: if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL)
2873: goto out;
2874:
2875: /* Get the current flags for this page. */
2876: simple_lock(&pg->mdpage.pvh_slock);
2877:
2878: pv = pmap_find_pv(pg, pm, va);
2879: if (pv == NULL) {
2880: simple_unlock(&pg->mdpage.pvh_slock);
2881: goto out;
2882: }
2883:
2884: pg->mdpage.pvh_attrs |= PVF_REF;
2885: pv->pv_flags |= PVF_REF;
2886: simple_unlock(&pg->mdpage.pvh_slock);
1.1 matt 2887:
1.134 thorpej 2888: NPDEBUG(PDB_FOLLOW,
2889: printf("pmap_fault_fixup: ref emul. pm %p, va 0x%08lx, pa 0x%08lx\n",
1.155 yamt 2890: pm, va, VM_PAGE_TO_PHYS(pg)));
1.134 thorpej 2891:
2892: *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO;
2893: PTE_SYNC(ptep);
2894: rv = 1;
2895: }
2896:
2897: /*
2898: * We know there is a valid mapping here, so simply
2899: * fix up the L1 if necessary.
2900: */
2901: pl1pd = &pm->pm_l1->l1_kva[l1idx];
2902: l1pd = l2b->l2b_phys | L1_C_DOM(pm->pm_domain) | L1_C_PROTO;
2903: if (*pl1pd != l1pd) {
2904: *pl1pd = l1pd;
2905: PTE_SYNC(pl1pd);
2906: rv = 1;
2907: }
2908:
2909: #ifdef CPU_SA110
2910: /*
2911: * There are bugs in the rev K SA110. This is a check for one
2912: * of them.
2913: */
2914: if (rv == 0 && curcpu()->ci_arm_cputype == CPU_ID_SA110 &&
2915: curcpu()->ci_arm_cpurev < 3) {
2916: /* Always current pmap */
2917: if (l2pte_valid(pte)) {
2918: extern int kernel_debug;
2919: if (kernel_debug & 1) {
2920: struct proc *p = curlwp->l_proc;
2921: printf("prefetch_abort: page is already "
2922: "mapped - pte=%p *pte=%08x\n", ptep, pte);
2923: printf("prefetch_abort: pc=%08lx proc=%p "
2924: "process=%s\n", va, p, p->p_comm);
2925: printf("prefetch_abort: far=%08x fs=%x\n",
2926: cpu_faultaddress(), cpu_faultstatus());
1.113 thorpej 2927: }
1.134 thorpej 2928: #ifdef DDB
2929: if (kernel_debug & 2)
2930: Debugger();
2931: #endif
2932: rv = 1;
1.1 matt 2933: }
2934: }
1.134 thorpej 2935: #endif /* CPU_SA110 */
1.104 thorpej 2936:
1.134 thorpej 2937: #ifdef DEBUG
2938: /*
2939: * If 'rv == 0' at this point, it generally indicates that there is a
2940: * stale TLB entry for the faulting address. This happens when two or
2941: * more processes are sharing an L1. Since we don't flush the TLB on
2942: * a context switch between such processes, we can take domain faults
2943: * for mappings which exist at the same VA in both processes. EVEN IF
2944: * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for
2945: * example.
2946: *
2947: * This is extremely likely to happen if pmap_enter() updated the L1
2948: * entry for a recently entered mapping. In this case, the TLB is
2949: * flushed for the new mapping, but there may still be TLB entries for
2950: * other mappings belonging to other processes in the 1MB range
2951: * covered by the L1 entry.
2952: *
2953: * Since 'rv == 0', we know that the L1 already contains the correct
2954: * value, so the fault must be due to a stale TLB entry.
2955: *
2956: * Since we always need to flush the TLB anyway in the case where we
2957: * fixed up the L1, or frobbed the L2 PTE, we effectively deal with
2958: * stale TLB entries dynamically.
2959: *
2960: * However, the above condition can ONLY happen if the current L1 is
2961: * being shared. If it happens when the L1 is unshared, it indicates
2962: * that other parts of the pmap are not doing their job WRT managing
2963: * the TLB.
2964: */
2965: if (rv == 0 && pm->pm_l1->l1_domain_use_count == 1) {
2966: extern int last_fault_code;
2967: printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n",
2968: pm, va, ftype);
2969: printf("fixup: l2 %p, l2b %p, ptep %p, pl1pd %p\n",
2970: l2, l2b, ptep, pl1pd);
2971: printf("fixup: pte 0x%x, l1pd 0x%x, last code 0x%x\n",
2972: pte, l1pd, last_fault_code);
2973: #ifdef DDB
2974: Debugger();
2975: #endif
2976: }
2977: #endif
2978:
2979: cpu_tlb_flushID_SE(va);
2980: cpu_cpwait();
2981:
2982: rv = 1;
1.104 thorpej 2983:
1.134 thorpej 2984: out:
2985: pmap_release_pmap_lock(pm);
1.17 chris 2986: PMAP_MAP_TO_HEAD_UNLOCK();
1.134 thorpej 2987:
2988: return (rv);
2989: }
2990:
2991: /*
2992: * pmap_collect: free resources held by a pmap
2993: *
2994: * => optional function.
2995: * => called when a process is swapped out to free memory.
2996: */
2997: void
2998: pmap_collect(pmap_t pm)
2999: {
1.156 scw 3000:
3001: pmap_idcache_wbinv_all(pm);
1.160 thorpej 3002: pm->pm_remove_all = true;
1.156 scw 3003: pmap_do_remove(pm, VM_MIN_ADDRESS, VM_MAX_ADDRESS, 1);
3004: pmap_update(pm);
1.1 matt 3005: }
3006:
3007: /*
1.134 thorpej 3008: * Routine: pmap_procwr
3009: *
1.1 matt 3010: * Function:
1.134 thorpej 3011: * Synchronize caches corresponding to [addr, addr+len) in p.
3012: *
3013: */
3014: void
3015: pmap_procwr(struct proc *p, vaddr_t va, int len)
3016: {
3017: /* We only need to do anything if it is the current process. */
3018: if (p == curproc)
3019: cpu_icache_sync_range(va, len);
3020: }
3021:
3022: /*
3023: * Routine: pmap_unwire
3024: * Function: Clear the wired attribute for a map/virtual-address pair.
3025: *
3026: * In/out conditions:
3027: * The mapping must already exist in the pmap.
1.1 matt 3028: */
1.134 thorpej 3029: void
3030: pmap_unwire(pmap_t pm, vaddr_t va)
3031: {
3032: struct l2_bucket *l2b;
3033: pt_entry_t *ptep, pte;
3034: struct vm_page *pg;
3035: paddr_t pa;
3036:
3037: NPDEBUG(PDB_WIRING, printf("pmap_unwire: pm %p, va 0x%08lx\n", pm, va));
3038:
3039: PMAP_MAP_TO_HEAD_LOCK();
3040: pmap_acquire_pmap_lock(pm);
3041:
3042: l2b = pmap_get_l2_bucket(pm, va);
3043: KDASSERT(l2b != NULL);
3044:
3045: ptep = &l2b->l2b_kva[l2pte_index(va)];
3046: pte = *ptep;
3047:
3048: /* Extract the physical address of the page */
3049: pa = l2pte_pa(pte);
1.1 matt 3050:
1.134 thorpej 3051: if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
3052: /* Update the wired bit in the pv entry for this page. */
3053: simple_lock(&pg->mdpage.pvh_slock);
3054: (void) pmap_modify_pv(pg, pm, va, PVF_WIRED, 0);
3055: simple_unlock(&pg->mdpage.pvh_slock);
3056: }
3057:
3058: pmap_release_pmap_lock(pm);
3059: PMAP_MAP_TO_HEAD_UNLOCK();
3060: }
3061:
3062: void
1.163.4.2! garbled 3063: pmap_switch(struct lwp *olwp, struct lwp *nlwp)
1.1 matt 3064: {
1.163.4.2! garbled 3065: extern int block_userspace_access;
! 3066: pmap_t opm, npm, rpm;
! 3067: uint32_t odacr, ndacr;
! 3068: int oldirqstate;
1.134 thorpej 3069:
1.163.4.2! garbled 3070: npm = nlwp->l_proc->p_vmspace->vm_map.pmap;
! 3071: ndacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) |
! 3072: (DOMAIN_CLIENT << (npm->pm_domain * 2));
1.134 thorpej 3073:
1.163.4.2! garbled 3074: /*
! 3075: * If TTB and DACR are unchanged, short-circuit all the
! 3076: * TLB/cache management stuff.
! 3077: */
! 3078: if (olwp != NULL) {
! 3079: opm = olwp->l_proc->p_vmspace->vm_map.pmap;
! 3080: odacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) |
! 3081: (DOMAIN_CLIENT << (opm->pm_domain * 2));
1.134 thorpej 3082:
1.163.4.2! garbled 3083: if (opm->pm_l1 == npm->pm_l1 && odacr == ndacr)
! 3084: goto all_done;
! 3085: } else
! 3086: opm = NULL;
1.1 matt 3087:
1.163.4.2! garbled 3088: block_userspace_access = 1;
1.1 matt 3089:
1.163.4.2! garbled 3090: /*
! 3091: * If switching to a user vmspace which is different to the
! 3092: * most recent one, and the most recent one is potentially
! 3093: * live in the cache, we must write-back and invalidate the
! 3094: * entire cache.
! 3095: */
! 3096: rpm = pmap_recent_user;
! 3097: if (npm != pmap_kernel() && rpm && npm != rpm &&
! 3098: rpm->pm_cstate.cs_cache) {
! 3099: rpm->pm_cstate.cs_cache = 0;
! 3100: cpu_idcache_wbinv_all();
! 3101: }
1.1 matt 3102:
1.163.4.2! garbled 3103: /* No interrupts while we frob the TTB/DACR */
! 3104: oldirqstate = disable_interrupts(I32_bit | F32_bit);
1.1 matt 3105:
1.163.4.2! garbled 3106: /*
! 3107: * For ARM_VECTORS_LOW, we MUST, I repeat, MUST fix up the L1
! 3108: * entry corresponding to 'vector_page' in the incoming L1 table
! 3109: * before switching to it otherwise subsequent interrupts/exceptions
! 3110: * (including domain faults!) will jump into hyperspace.
! 3111: */
! 3112: if (npm->pm_pl1vec != NULL) {
! 3113: cpu_tlb_flushID_SE((u_int)vector_page);
! 3114: cpu_cpwait();
! 3115: *npm->pm_pl1vec = npm->pm_l1vec;
! 3116: PTE_SYNC(npm->pm_pl1vec);
! 3117: }
1.1 matt 3118:
1.163.4.2! garbled 3119: cpu_domains(ndacr);
1.1 matt 3120:
1.163.4.2! garbled 3121: if (npm == pmap_kernel() || npm == rpm) {
1.1 matt 3122: /*
1.163.4.2! garbled 3123: * Switching to a kernel thread, or back to the
! 3124: * same user vmspace as before... Simply update
! 3125: * the TTB (no TLB flush required)
1.1 matt 3126: */
1.163.4.2! garbled 3127: __asm volatile("mcr p15, 0, %0, c2, c0, 0" ::
! 3128: "r"(npm->pm_l1->l1_physaddr));
! 3129: cpu_cpwait();
! 3130: } else {
1.1 matt 3131: /*
1.163.4.2! garbled 3132: * Otherwise, update TTB and flush TLB
1.1 matt 3133: */
1.163.4.2! garbled 3134: cpu_context_switch(npm->pm_l1->l1_physaddr);
! 3135: if (rpm != NULL)
! 3136: rpm->pm_cstate.cs_tlb = 0;
1.1 matt 3137: }
1.163.4.2! garbled 3138:
! 3139: restore_interrupts(oldirqstate);
! 3140:
! 3141: block_userspace_access = 0;
! 3142:
! 3143: all_done:
! 3144: /*
! 3145: * The new pmap is resident. Make sure it's marked
! 3146: * as resident in the cache/TLB.
! 3147: */
! 3148: npm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL;
! 3149: if (npm != pmap_kernel())
! 3150: pmap_recent_user = npm;
! 3151:
! 3152: /* The old pmap is not longer active */
! 3153: if (opm != NULL)
! 3154: opm->pm_activated = false;
! 3155:
! 3156: /* But the new one is */
! 3157: npm->pm_activated = true;
! 3158: }
! 3159:
! 3160: void
! 3161: pmap_activate(struct lwp *l)
! 3162: {
! 3163:
! 3164: if (l == curlwp &&
! 3165: l->l_proc->p_vmspace->vm_map.pmap->pm_activated == false)
! 3166: pmap_switch(NULL, l);
1.134 thorpej 3167: }
1.1 matt 3168:
1.134 thorpej 3169: void
3170: pmap_deactivate(struct lwp *l)
3171: {
1.163.4.2! garbled 3172:
! 3173: l->l_proc->p_vmspace->vm_map.pmap->pm_activated = false;
1.1 matt 3174: }
3175:
3176: void
1.134 thorpej 3177: pmap_update(pmap_t pm)
1.1 matt 3178: {
3179:
1.134 thorpej 3180: if (pm->pm_remove_all) {
3181: /*
3182: * Finish up the pmap_remove_all() optimisation by flushing
3183: * the TLB.
3184: */
3185: pmap_tlb_flushID(pm);
1.160 thorpej 3186: pm->pm_remove_all = false;
1.134 thorpej 3187: }
1.1 matt 3188:
1.134 thorpej 3189: if (pmap_is_current(pm)) {
1.107 thorpej 3190: /*
1.134 thorpej 3191: * If we're dealing with a current userland pmap, move its L1
3192: * to the end of the LRU.
1.107 thorpej 3193: */
1.134 thorpej 3194: if (pm != pmap_kernel())
3195: pmap_use_l1(pm);
3196:
1.1 matt 3197: /*
1.134 thorpej 3198: * We can assume we're done with frobbing the cache/tlb for
3199: * now. Make sure any future pmap ops don't skip cache/tlb
3200: * flushes.
1.1 matt 3201: */
1.134 thorpej 3202: pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL;
1.1 matt 3203: }
3204:
1.96 thorpej 3205: /*
1.134 thorpej 3206: * make sure TLB/cache operations have completed.
1.96 thorpej 3207: */
1.134 thorpej 3208: cpu_cpwait();
3209: }
3210:
3211: void
3212: pmap_remove_all(pmap_t pm)
3213: {
1.96 thorpej 3214:
1.1 matt 3215: /*
1.134 thorpej 3216: * The vmspace described by this pmap is about to be torn down.
3217: * Until pmap_update() is called, UVM will only make calls
3218: * to pmap_remove(). We can make life much simpler by flushing
3219: * the cache now, and deferring TLB invalidation to pmap_update().
1.1 matt 3220: */
1.134 thorpej 3221: pmap_idcache_wbinv_all(pm);
1.160 thorpej 3222: pm->pm_remove_all = true;
1.1 matt 3223: }
3224:
3225: /*
1.134 thorpej 3226: * Retire the given physical map from service.
3227: * Should only be called if the map contains no valid mappings.
1.1 matt 3228: */
1.134 thorpej 3229: void
3230: pmap_destroy(pmap_t pm)
1.1 matt 3231: {
1.134 thorpej 3232: u_int count;
1.1 matt 3233:
1.134 thorpej 3234: if (pm == NULL)
3235: return;
1.1 matt 3236:
1.134 thorpej 3237: if (pm->pm_remove_all) {
3238: pmap_tlb_flushID(pm);
1.160 thorpej 3239: pm->pm_remove_all = false;
1.1 matt 3240: }
1.79 thorpej 3241:
1.49 thorpej 3242: /*
1.134 thorpej 3243: * Drop reference count
1.49 thorpej 3244: */
1.134 thorpej 3245: simple_lock(&pm->pm_lock);
3246: count = --pm->pm_obj.uo_refs;
3247: simple_unlock(&pm->pm_lock);
3248: if (count > 0) {
3249: if (pmap_is_current(pm)) {
3250: if (pm != pmap_kernel())
3251: pmap_use_l1(pm);
3252: pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL;
3253: }
3254: return;
3255: }
1.66 thorpej 3256:
1.1 matt 3257: /*
1.134 thorpej 3258: * reference count is zero, free pmap resources and then free pmap.
1.1 matt 3259: */
1.134 thorpej 3260:
3261: if (vector_page < KERNEL_BASE) {
1.163.4.2! garbled 3262: KDASSERT(!pmap_is_current(pm));
1.147 scw 3263:
1.134 thorpej 3264: /* Remove the vector page mapping */
3265: pmap_remove(pm, vector_page, vector_page + PAGE_SIZE);
3266: pmap_update(pm);
1.1 matt 3267: }
3268:
1.134 thorpej 3269: LIST_REMOVE(pm, pm_list);
3270:
3271: pmap_free_l1(pm);
3272:
1.163.4.2! garbled 3273: if (pmap_recent_user == pm)
! 3274: pmap_recent_user = NULL;
! 3275:
1.134 thorpej 3276: /* return the pmap to the pool */
3277: pool_cache_put(&pmap_pmap_cache, pm);
3278: }
3279:
3280:
3281: /*
3282: * void pmap_reference(pmap_t pm)
3283: *
3284: * Add a reference to the specified pmap.
3285: */
3286: void
3287: pmap_reference(pmap_t pm)
3288: {
1.1 matt 3289:
1.134 thorpej 3290: if (pm == NULL)
3291: return;
1.1 matt 3292:
1.134 thorpej 3293: pmap_use_l1(pm);
1.104 thorpej 3294:
1.134 thorpej 3295: simple_lock(&pm->pm_lock);
3296: pm->pm_obj.uo_refs++;
3297: simple_unlock(&pm->pm_lock);
3298: }
1.49 thorpej 3299:
1.134 thorpej 3300: /*
3301: * pmap_zero_page()
3302: *
3303: * Zero a given physical page by mapping it at a page hook point.
3304: * In doing the zero page op, the page we zero is mapped cachable, as with
3305: * StrongARM accesses to non-cached pages are non-burst making writing
3306: * _any_ bulk data very slow.
3307: */
3308: #if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0
3309: void
3310: pmap_zero_page_generic(paddr_t phys)
3311: {
3312: #ifdef DEBUG
3313: struct vm_page *pg = PHYS_TO_VM_PAGE(phys);
1.1 matt 3314:
1.134 thorpej 3315: if (pg->mdpage.pvh_list != NULL)
3316: panic("pmap_zero_page: page has mappings");
3317: #endif
1.1 matt 3318:
1.134 thorpej 3319: KDASSERT((phys & PGOFSET) == 0);
1.120 chris 3320:
1.134 thorpej 3321: /*
3322: * Hook in the page, zero it, and purge the cache for that
3323: * zeroed page. Invalidate the TLB as needed.
3324: */
3325: *cdst_pte = L2_S_PROTO | phys |
3326: L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode;
3327: PTE_SYNC(cdst_pte);
3328: cpu_tlb_flushD_SE(cdstp);
3329: cpu_cpwait();
3330: bzero_page(cdstp);
3331: cpu_dcache_wbinv_range(cdstp, PAGE_SIZE);
3332: }
3333: #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */
1.1 matt 3334:
1.134 thorpej 3335: #if ARM_MMU_XSCALE == 1
3336: void
3337: pmap_zero_page_xscale(paddr_t phys)
3338: {
3339: #ifdef DEBUG
3340: struct vm_page *pg = PHYS_TO_VM_PAGE(phys);
1.1 matt 3341:
1.134 thorpej 3342: if (pg->mdpage.pvh_list != NULL)
3343: panic("pmap_zero_page: page has mappings");
3344: #endif
1.1 matt 3345:
1.134 thorpej 3346: KDASSERT((phys & PGOFSET) == 0);
1.1 matt 3347:
1.134 thorpej 3348: /*
3349: * Hook in the page, zero it, and purge the cache for that
3350: * zeroed page. Invalidate the TLB as needed.
3351: */
3352: *cdst_pte = L2_S_PROTO | phys |
3353: L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) |
3354: L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */
3355: PTE_SYNC(cdst_pte);
3356: cpu_tlb_flushD_SE(cdstp);
3357: cpu_cpwait();
3358: bzero_page(cdstp);
3359: xscale_cache_clean_minidata();
3360: }
3361: #endif /* ARM_MMU_XSCALE == 1 */
1.1 matt 3362:
1.134 thorpej 3363: /* pmap_pageidlezero()
3364: *
3365: * The same as above, except that we assume that the page is not
3366: * mapped. This means we never have to flush the cache first. Called
3367: * from the idle loop.
3368: */
1.159 thorpej 3369: bool
1.134 thorpej 3370: pmap_pageidlezero(paddr_t phys)
3371: {
3372: unsigned int i;
3373: int *ptr;
1.160 thorpej 3374: bool rv = true;
1.134 thorpej 3375: #ifdef DEBUG
3376: struct vm_page *pg;
3377:
3378: pg = PHYS_TO_VM_PAGE(phys);
3379: if (pg->mdpage.pvh_list != NULL)
3380: panic("pmap_pageidlezero: page has mappings");
1.1 matt 3381: #endif
3382:
1.134 thorpej 3383: KDASSERT((phys & PGOFSET) == 0);
3384:
1.109 thorpej 3385: /*
1.134 thorpej 3386: * Hook in the page, zero it, and purge the cache for that
3387: * zeroed page. Invalidate the TLB as needed.
1.109 thorpej 3388: */
1.134 thorpej 3389: *cdst_pte = L2_S_PROTO | phys |
3390: L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode;
3391: PTE_SYNC(cdst_pte);
3392: cpu_tlb_flushD_SE(cdstp);
3393: cpu_cpwait();
1.1 matt 3394:
1.134 thorpej 3395: for (i = 0, ptr = (int *)cdstp;
3396: i < (PAGE_SIZE / sizeof(int)); i++) {
1.163.4.1 matt 3397: if (sched_curcpu_runnable_p()) {
1.134 thorpej 3398: /*
3399: * A process has become ready. Abort now,
3400: * so we don't keep it waiting while we
3401: * do slow memory access to finish this
3402: * page.
3403: */
1.160 thorpej 3404: rv = false;
1.134 thorpej 3405: break;
3406: }
3407: *ptr++ = 0;
1.11 chris 3408: }
1.1 matt 3409:
1.134 thorpej 3410: if (rv)
3411: /*
3412: * if we aborted we'll rezero this page again later so don't
3413: * purge it unless we finished it
3414: */
3415: cpu_dcache_wbinv_range(cdstp, PAGE_SIZE);
1.1 matt 3416:
1.134 thorpej 3417: return (rv);
1.1 matt 3418: }
1.134 thorpej 3419:
1.48 chris 3420: /*
1.134 thorpej 3421: * pmap_copy_page()
1.48 chris 3422: *
1.134 thorpej 3423: * Copy one physical page into another, by mapping the pages into
3424: * hook points. The same comment regarding cachability as in
3425: * pmap_zero_page also applies here.
1.48 chris 3426: */
1.134 thorpej 3427: #if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0
1.1 matt 3428: void
1.134 thorpej 3429: pmap_copy_page_generic(paddr_t src, paddr_t dst)
1.1 matt 3430: {
1.134 thorpej 3431: struct vm_page *src_pg = PHYS_TO_VM_PAGE(src);
3432: #ifdef DEBUG
3433: struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst);
1.105 thorpej 3434:
1.134 thorpej 3435: if (dst_pg->mdpage.pvh_list != NULL)
3436: panic("pmap_copy_page: dst page has mappings");
3437: #endif
1.83 thorpej 3438:
1.134 thorpej 3439: KDASSERT((src & PGOFSET) == 0);
3440: KDASSERT((dst & PGOFSET) == 0);
1.105 thorpej 3441:
1.134 thorpej 3442: /*
3443: * Clean the source page. Hold the source page's lock for
3444: * the duration of the copy so that no other mappings can
3445: * be created while we have a potentially aliased mapping.
3446: */
3447: simple_lock(&src_pg->mdpage.pvh_slock);
1.160 thorpej 3448: (void) pmap_clean_page(src_pg->mdpage.pvh_list, true);
1.105 thorpej 3449:
1.134 thorpej 3450: /*
3451: * Map the pages into the page hook points, copy them, and purge
3452: * the cache for the appropriate page. Invalidate the TLB
3453: * as required.
3454: */
3455: *csrc_pte = L2_S_PROTO | src |
3456: L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode;
3457: PTE_SYNC(csrc_pte);
3458: *cdst_pte = L2_S_PROTO | dst |
3459: L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode;
3460: PTE_SYNC(cdst_pte);
3461: cpu_tlb_flushD_SE(csrcp);
3462: cpu_tlb_flushD_SE(cdstp);
3463: cpu_cpwait();
3464: bcopy_page(csrcp, cdstp);
3465: cpu_dcache_inv_range(csrcp, PAGE_SIZE);
3466: simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */
3467: cpu_dcache_wbinv_range(cdstp, PAGE_SIZE);
1.1 matt 3468: }
1.134 thorpej 3469: #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */
1.1 matt 3470:
1.134 thorpej 3471: #if ARM_MMU_XSCALE == 1
1.1 matt 3472: void
1.134 thorpej 3473: pmap_copy_page_xscale(paddr_t src, paddr_t dst)
1.1 matt 3474: {
1.134 thorpej 3475: struct vm_page *src_pg = PHYS_TO_VM_PAGE(src);
3476: #ifdef DEBUG
3477: struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst);
1.14 chs 3478:
1.134 thorpej 3479: if (dst_pg->mdpage.pvh_list != NULL)
3480: panic("pmap_copy_page: dst page has mappings");
3481: #endif
1.13 chris 3482:
1.134 thorpej 3483: KDASSERT((src & PGOFSET) == 0);
3484: KDASSERT((dst & PGOFSET) == 0);
1.14 chs 3485:
1.134 thorpej 3486: /*
3487: * Clean the source page. Hold the source page's lock for
3488: * the duration of the copy so that no other mappings can
3489: * be created while we have a potentially aliased mapping.
3490: */
3491: simple_lock(&src_pg->mdpage.pvh_slock);
1.160 thorpej 3492: (void) pmap_clean_page(src_pg->mdpage.pvh_list, true);
1.105 thorpej 3493:
1.134 thorpej 3494: /*
3495: * Map the pages into the page hook points, copy them, and purge
3496: * the cache for the appropriate page. Invalidate the TLB
3497: * as required.
3498: */
3499: *csrc_pte = L2_S_PROTO | src |
3500: L2_S_PROT(PTE_KERNEL, VM_PROT_READ) |
3501: L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */
3502: PTE_SYNC(csrc_pte);
3503: *cdst_pte = L2_S_PROTO | dst |
3504: L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) |
3505: L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */
3506: PTE_SYNC(cdst_pte);
3507: cpu_tlb_flushD_SE(csrcp);
3508: cpu_tlb_flushD_SE(cdstp);
3509: cpu_cpwait();
3510: bcopy_page(csrcp, cdstp);
3511: simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */
3512: xscale_cache_clean_minidata();
1.1 matt 3513: }
1.134 thorpej 3514: #endif /* ARM_MMU_XSCALE == 1 */
1.1 matt 3515:
3516: /*
1.134 thorpej 3517: * void pmap_virtual_space(vaddr_t *start, vaddr_t *end)
1.1 matt 3518: *
1.134 thorpej 3519: * Return the start and end addresses of the kernel's virtual space.
3520: * These values are setup in pmap_bootstrap and are updated as pages
3521: * are allocated.
1.1 matt 3522: */
3523: void
1.134 thorpej 3524: pmap_virtual_space(vaddr_t *start, vaddr_t *end)
1.1 matt 3525: {
1.134 thorpej 3526: *start = virtual_avail;
3527: *end = virtual_end;
1.1 matt 3528: }
3529:
3530: /*
1.134 thorpej 3531: * Helper function for pmap_grow_l2_bucket()
1.1 matt 3532: */
1.157 perry 3533: static inline int
1.134 thorpej 3534: pmap_grow_map(vaddr_t va, pt_entry_t cache_mode, paddr_t *pap)
1.1 matt 3535: {
1.134 thorpej 3536: struct l2_bucket *l2b;
3537: pt_entry_t *ptep;
1.2 matt 3538: paddr_t pa;
1.1 matt 3539:
1.160 thorpej 3540: if (uvm.page_init_done == false) {
3541: if (uvm_page_physget(&pa) == false)
1.134 thorpej 3542: return (1);
3543: } else {
3544: struct vm_page *pg;
3545: pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
3546: if (pg == NULL)
3547: return (1);
3548: pa = VM_PAGE_TO_PHYS(pg);
3549: }
1.1 matt 3550:
1.134 thorpej 3551: if (pap)
3552: *pap = pa;
1.1 matt 3553:
1.134 thorpej 3554: l2b = pmap_get_l2_bucket(pmap_kernel(), va);
3555: KDASSERT(l2b != NULL);
1.1 matt 3556:
1.134 thorpej 3557: ptep = &l2b->l2b_kva[l2pte_index(va)];
3558: *ptep = L2_S_PROTO | pa | cache_mode |
3559: L2_S_PROT(PTE_KERNEL, VM_PROT_READ | VM_PROT_WRITE);
3560: PTE_SYNC(ptep);
3561: memset((void *)va, 0, PAGE_SIZE);
3562: return (0);
1.1 matt 3563: }
3564:
3565: /*
1.134 thorpej 3566: * This is the same as pmap_alloc_l2_bucket(), except that it is only
3567: * used by pmap_growkernel().
1.1 matt 3568: */
1.157 perry 3569: static inline struct l2_bucket *
1.134 thorpej 3570: pmap_grow_l2_bucket(pmap_t pm, vaddr_t va)
1.1 matt 3571: {
1.134 thorpej 3572: struct l2_dtable *l2;
3573: struct l2_bucket *l2b;
3574: u_short l1idx;
3575: vaddr_t nva;
3576:
3577: l1idx = L1_IDX(va);
3578:
3579: if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) {
3580: /*
3581: * No mapping at this address, as there is
3582: * no entry in the L1 table.
3583: * Need to allocate a new l2_dtable.
3584: */
3585: nva = pmap_kernel_l2dtable_kva;
3586: if ((nva & PGOFSET) == 0) {
3587: /*
3588: * Need to allocate a backing page
3589: */
3590: if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL))
3591: return (NULL);
3592: }
1.1 matt 3593:
1.134 thorpej 3594: l2 = (struct l2_dtable *)nva;
3595: nva += sizeof(struct l2_dtable);
1.82 thorpej 3596:
1.134 thorpej 3597: if ((nva & PGOFSET) < (pmap_kernel_l2dtable_kva & PGOFSET)) {
3598: /*
3599: * The new l2_dtable straddles a page boundary.
3600: * Map in another page to cover it.
3601: */
3602: if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL))
3603: return (NULL);
3604: }
1.1 matt 3605:
1.134 thorpej 3606: pmap_kernel_l2dtable_kva = nva;
1.1 matt 3607:
1.134 thorpej 3608: /*
3609: * Link it into the parent pmap
3610: */
3611: pm->pm_l2[L2_IDX(l1idx)] = l2;
1.82 thorpej 3612: }
1.75 reinoud 3613:
1.134 thorpej 3614: l2b = &l2->l2_bucket[L2_BUCKET(l1idx)];
3615:
3616: /*
3617: * Fetch pointer to the L2 page table associated with the address.
3618: */
3619: if (l2b->l2b_kva == NULL) {
3620: pt_entry_t *ptep;
3621:
3622: /*
3623: * No L2 page table has been allocated. Chances are, this
3624: * is because we just allocated the l2_dtable, above.
3625: */
3626: nva = pmap_kernel_l2ptp_kva;
3627: ptep = (pt_entry_t *)nva;
3628: if ((nva & PGOFSET) == 0) {
3629: /*
3630: * Need to allocate a backing page
3631: */
3632: if (pmap_grow_map(nva, pte_l2_s_cache_mode_pt,
3633: &pmap_kernel_l2ptp_phys))
3634: return (NULL);
3635: PTE_SYNC_RANGE(ptep, PAGE_SIZE / sizeof(pt_entry_t));
3636: }
3637:
3638: l2->l2_occupancy++;
3639: l2b->l2b_kva = ptep;
3640: l2b->l2b_l1idx = l1idx;
3641: l2b->l2b_phys = pmap_kernel_l2ptp_phys;
3642:
3643: pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL;
3644: pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL;
1.82 thorpej 3645: }
1.1 matt 3646:
1.134 thorpej 3647: return (l2b);
3648: }
3649:
3650: vaddr_t
3651: pmap_growkernel(vaddr_t maxkvaddr)
3652: {
3653: pmap_t kpm = pmap_kernel();
3654: struct l1_ttable *l1;
3655: struct l2_bucket *l2b;
3656: pd_entry_t *pl1pd;
3657: int s;
3658:
3659: if (maxkvaddr <= pmap_curmaxkvaddr)
3660: goto out; /* we are OK */
1.1 matt 3661:
1.134 thorpej 3662: NPDEBUG(PDB_GROWKERN,
3663: printf("pmap_growkernel: growing kernel from 0x%lx to 0x%lx\n",
3664: pmap_curmaxkvaddr, maxkvaddr));
1.1 matt 3665:
1.134 thorpej 3666: KDASSERT(maxkvaddr <= virtual_end);
1.34 thorpej 3667:
1.134 thorpej 3668: /*
3669: * whoops! we need to add kernel PTPs
3670: */
1.1 matt 3671:
1.134 thorpej 3672: s = splhigh(); /* to be safe */
3673: simple_lock(&kpm->pm_lock);
1.1 matt 3674:
1.134 thorpej 3675: /* Map 1MB at a time */
3676: for (; pmap_curmaxkvaddr < maxkvaddr; pmap_curmaxkvaddr += L1_S_SIZE) {
1.1 matt 3677:
1.134 thorpej 3678: l2b = pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr);
3679: KDASSERT(l2b != NULL);
1.1 matt 3680:
1.134 thorpej 3681: /* Distribute new L1 entry to all other L1s */
3682: SLIST_FOREACH(l1, &l1_list, l1_link) {
3683: pl1pd = &l1->l1_kva[L1_IDX(pmap_curmaxkvaddr)];
3684: *pl1pd = l2b->l2b_phys | L1_C_DOM(PMAP_DOMAIN_KERNEL) |
3685: L1_C_PROTO;
3686: PTE_SYNC(pl1pd);
3687: }
1.1 matt 3688: }
3689:
1.134 thorpej 3690: /*
3691: * flush out the cache, expensive but growkernel will happen so
3692: * rarely
3693: */
3694: cpu_dcache_wbinv_all();
3695: cpu_tlb_flushD();
3696: cpu_cpwait();
3697:
3698: simple_unlock(&kpm->pm_lock);
3699: splx(s);
1.1 matt 3700:
1.134 thorpej 3701: out:
3702: return (pmap_curmaxkvaddr);
1.1 matt 3703: }
3704:
1.134 thorpej 3705: /************************ Utility routines ****************************/
1.1 matt 3706:
1.134 thorpej 3707: /*
3708: * vector_page_setprot:
3709: *
3710: * Manipulate the protection of the vector page.
3711: */
3712: void
3713: vector_page_setprot(int prot)
1.11 chris 3714: {
1.134 thorpej 3715: struct l2_bucket *l2b;
3716: pt_entry_t *ptep;
3717:
3718: l2b = pmap_get_l2_bucket(pmap_kernel(), vector_page);
3719: KDASSERT(l2b != NULL);
1.17 chris 3720:
1.134 thorpej 3721: ptep = &l2b->l2b_kva[l2pte_index(vector_page)];
1.72 thorpej 3722:
1.134 thorpej 3723: *ptep = (*ptep & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot);
3724: PTE_SYNC(ptep);
3725: cpu_tlb_flushD_SE(vector_page);
1.32 thorpej 3726: cpu_cpwait();
1.17 chris 3727: }
3728:
3729: /*
1.134 thorpej 3730: * Fetch pointers to the PDE/PTE for the given pmap/VA pair.
1.160 thorpej 3731: * Returns true if the mapping exists, else false.
1.134 thorpej 3732: *
3733: * NOTE: This function is only used by a couple of arm-specific modules.
3734: * It is not safe to take any pmap locks here, since we could be right
3735: * in the middle of debugging the pmap anyway...
3736: *
1.160 thorpej 3737: * It is possible for this routine to return false even though a valid
1.134 thorpej 3738: * mapping does exist. This is because we don't lock, so the metadata
3739: * state may be inconsistent.
3740: *
3741: * NOTE: We can return a NULL *ptp in the case where the L1 pde is
3742: * a "section" mapping.
1.1 matt 3743: */
1.159 thorpej 3744: bool
1.134 thorpej 3745: pmap_get_pde_pte(pmap_t pm, vaddr_t va, pd_entry_t **pdp, pt_entry_t **ptp)
1.1 matt 3746: {
1.134 thorpej 3747: struct l2_dtable *l2;
3748: pd_entry_t *pl1pd, l1pd;
3749: pt_entry_t *ptep;
3750: u_short l1idx;
3751:
3752: if (pm->pm_l1 == NULL)
1.160 thorpej 3753: return (false);
1.134 thorpej 3754:
3755: l1idx = L1_IDX(va);
3756: *pdp = pl1pd = &pm->pm_l1->l1_kva[l1idx];
3757: l1pd = *pl1pd;
1.1 matt 3758:
1.134 thorpej 3759: if (l1pte_section_p(l1pd)) {
3760: *ptp = NULL;
1.160 thorpej 3761: return (true);
1.1 matt 3762: }
3763:
1.134 thorpej 3764: if (pm->pm_l2 == NULL)
1.160 thorpej 3765: return (false);
1.21 chris 3766:
1.134 thorpej 3767: l2 = pm->pm_l2[L2_IDX(l1idx)];
1.104 thorpej 3768:
1.134 thorpej 3769: if (l2 == NULL ||
3770: (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) {
1.160 thorpej 3771: return (false);
1.29 rearnsha 3772: }
1.21 chris 3773:
1.134 thorpej 3774: *ptp = &ptep[l2pte_index(va)];
1.160 thorpej 3775: return (true);
1.1 matt 3776: }
3777:
1.159 thorpej 3778: bool
1.134 thorpej 3779: pmap_get_pde(pmap_t pm, vaddr_t va, pd_entry_t **pdp)
1.1 matt 3780: {
1.134 thorpej 3781: u_short l1idx;
1.1 matt 3782:
1.134 thorpej 3783: if (pm->pm_l1 == NULL)
1.160 thorpej 3784: return (false);
1.50 thorpej 3785:
1.134 thorpej 3786: l1idx = L1_IDX(va);
3787: *pdp = &pm->pm_l1->l1_kva[l1idx];
1.50 thorpej 3788:
1.160 thorpej 3789: return (true);
1.1 matt 3790: }
3791:
1.134 thorpej 3792: /************************ Bootstrapping routines ****************************/
3793:
3794: static void
3795: pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt)
1.1 matt 3796: {
1.134 thorpej 3797: int i;
3798:
3799: l1->l1_kva = l1pt;
3800: l1->l1_domain_use_count = 0;
3801: l1->l1_domain_first = 0;
3802:
3803: for (i = 0; i < PMAP_DOMAINS; i++)
3804: l1->l1_domain_free[i] = i + 1;
1.1 matt 3805:
1.134 thorpej 3806: /*
3807: * Copy the kernel's L1 entries to each new L1.
3808: */
3809: if (pmap_initialized)
3810: memcpy(l1pt, pmap_kernel()->pm_l1->l1_kva, L1_TABLE_SIZE);
1.50 thorpej 3811:
1.134 thorpej 3812: if (pmap_extract(pmap_kernel(), (vaddr_t)l1pt,
1.160 thorpej 3813: &l1->l1_physaddr) == false)
1.134 thorpej 3814: panic("pmap_init_l1: can't get PA of L1 at %p", l1pt);
1.50 thorpej 3815:
1.134 thorpej 3816: SLIST_INSERT_HEAD(&l1_list, l1, l1_link);
3817: TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru);
1.1 matt 3818: }
3819:
1.50 thorpej 3820: /*
1.134 thorpej 3821: * pmap_bootstrap() is called from the board-specific initarm() routine
3822: * once the kernel L1/L2 descriptors tables have been set up.
3823: *
3824: * This is a somewhat convoluted process since pmap bootstrap is, effectively,
3825: * spread over a number of disparate files/functions.
1.50 thorpej 3826: *
1.134 thorpej 3827: * We are passed the following parameters
3828: * - kernel_l1pt
3829: * This is a pointer to the base of the kernel's L1 translation table.
3830: * - vstart
3831: * 1MB-aligned start of managed kernel virtual memory.
3832: * - vend
3833: * 1MB-aligned end of managed kernel virtual memory.
1.50 thorpej 3834: *
1.134 thorpej 3835: * We use the first parameter to build the metadata (struct l1_ttable and
3836: * struct l2_dtable) necessary to track kernel mappings.
1.50 thorpej 3837: */
1.134 thorpej 3838: #define PMAP_STATIC_L2_SIZE 16
3839: void
3840: pmap_bootstrap(pd_entry_t *kernel_l1pt, vaddr_t vstart, vaddr_t vend)
1.1 matt 3841: {
1.134 thorpej 3842: static struct l1_ttable static_l1;
3843: static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE];
3844: struct l1_ttable *l1 = &static_l1;
3845: struct l2_dtable *l2;
3846: struct l2_bucket *l2b;
3847: pmap_t pm = pmap_kernel();
3848: pd_entry_t pde;
3849: pt_entry_t *ptep;
1.2 matt 3850: paddr_t pa;
1.134 thorpej 3851: vaddr_t va;
3852: vsize_t size;
3853: int l1idx, l2idx, l2next = 0;
3854:
3855: /*
3856: * Initialise the kernel pmap object
3857: */
3858: pm->pm_l1 = l1;
3859: pm->pm_domain = PMAP_DOMAIN_KERNEL;
1.163.4.2! garbled 3860: pm->pm_activated = true;
1.134 thorpej 3861: pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL;
3862: simple_lock_init(&pm->pm_lock);
3863: pm->pm_obj.pgops = NULL;
3864: TAILQ_INIT(&pm->pm_obj.memq);
3865: pm->pm_obj.uo_npages = 0;
3866: pm->pm_obj.uo_refs = 1;
3867:
3868: /*
3869: * Scan the L1 translation table created by initarm() and create
3870: * the required metadata for all valid mappings found in it.
3871: */
3872: for (l1idx = 0; l1idx < (L1_TABLE_SIZE / sizeof(pd_entry_t)); l1idx++) {
3873: pde = kernel_l1pt[l1idx];
3874:
3875: /*
3876: * We're only interested in Coarse mappings.
3877: * pmap_extract() can deal with section mappings without
3878: * recourse to checking L2 metadata.
3879: */
3880: if ((pde & L1_TYPE_MASK) != L1_TYPE_C)
3881: continue;
3882:
3883: /*
3884: * Lookup the KVA of this L2 descriptor table
3885: */
3886: pa = (paddr_t)(pde & L1_C_ADDR_MASK);
3887: ptep = (pt_entry_t *)kernel_pt_lookup(pa);
3888: if (ptep == NULL) {
3889: panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx",
3890: (u_int)l1idx << L1_S_SHIFT, pa);
3891: }
3892:
3893: /*
3894: * Fetch the associated L2 metadata structure.
3895: * Allocate a new one if necessary.
3896: */
3897: if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) {
3898: if (l2next == PMAP_STATIC_L2_SIZE)
3899: panic("pmap_bootstrap: out of static L2s");
3900: pm->pm_l2[L2_IDX(l1idx)] = l2 = &static_l2[l2next++];
3901: }
3902:
3903: /*
3904: * One more L1 slot tracked...
3905: */
3906: l2->l2_occupancy++;
3907:
3908: /*
3909: * Fill in the details of the L2 descriptor in the
3910: * appropriate bucket.
3911: */
3912: l2b = &l2->l2_bucket[L2_BUCKET(l1idx)];
3913: l2b->l2b_kva = ptep;
3914: l2b->l2b_phys = pa;
3915: l2b->l2b_l1idx = l1idx;
1.1 matt 3916:
1.134 thorpej 3917: /*
3918: * Establish an initial occupancy count for this descriptor
3919: */
3920: for (l2idx = 0;
3921: l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t));
3922: l2idx++) {
3923: if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) {
3924: l2b->l2b_occupancy++;
3925: }
3926: }
1.1 matt 3927:
1.134 thorpej 3928: /*
3929: * Make sure the descriptor itself has the correct cache mode.
1.146 jdolecek 3930: * If not, fix it, but whine about the problem. Port-meisters
1.134 thorpej 3931: * should consider this a clue to fix up their initarm()
3932: * function. :)
3933: */
3934: if (pmap_set_pt_cache_mode(kernel_l1pt, (vaddr_t)ptep)) {
3935: printf("pmap_bootstrap: WARNING! wrong cache mode for "
3936: "L2 pte @ %p\n", ptep);
3937: }
3938: }
1.61 thorpej 3939:
1.134 thorpej 3940: /*
3941: * Ensure the primary (kernel) L1 has the correct cache mode for
3942: * a page table. Bitch if it is not correctly set.
3943: */
3944: for (va = (vaddr_t)kernel_l1pt;
3945: va < ((vaddr_t)kernel_l1pt + L1_TABLE_SIZE); va += PAGE_SIZE) {
3946: if (pmap_set_pt_cache_mode(kernel_l1pt, va))
3947: printf("pmap_bootstrap: WARNING! wrong cache mode for "
3948: "primary L1 @ 0x%lx\n", va);
1.1 matt 3949: }
3950:
1.134 thorpej 3951: cpu_dcache_wbinv_all();
3952: cpu_tlb_flushID();
3953: cpu_cpwait();
1.1 matt 3954:
1.113 thorpej 3955: /*
1.134 thorpej 3956: * now we allocate the "special" VAs which are used for tmp mappings
3957: * by the pmap (and other modules). we allocate the VAs by advancing
3958: * virtual_avail (note that there are no pages mapped at these VAs).
3959: *
3960: * Managed KVM space start from wherever initarm() tells us.
1.113 thorpej 3961: */
1.134 thorpej 3962: virtual_avail = vstart;
3963: virtual_end = vend;
1.113 thorpej 3964:
1.134 thorpej 3965: pmap_alloc_specials(&virtual_avail, 1, &csrcp, &csrc_pte);
3966: pmap_set_pt_cache_mode(kernel_l1pt, (vaddr_t)csrc_pte);
3967: pmap_alloc_specials(&virtual_avail, 1, &cdstp, &cdst_pte);
3968: pmap_set_pt_cache_mode(kernel_l1pt, (vaddr_t)cdst_pte);
1.139 matt 3969: pmap_alloc_specials(&virtual_avail, 1, (void *)&memhook, NULL);
1.134 thorpej 3970: pmap_alloc_specials(&virtual_avail, round_page(MSGBUFSIZE) / PAGE_SIZE,
1.139 matt 3971: (void *)&msgbufaddr, NULL);
1.134 thorpej 3972:
3973: /*
3974: * Allocate a range of kernel virtual address space to be used
3975: * for L2 descriptor tables and metadata allocation in
3976: * pmap_growkernel().
3977: */
3978: size = ((virtual_end - pmap_curmaxkvaddr) + L1_S_OFFSET) / L1_S_SIZE;
3979: pmap_alloc_specials(&virtual_avail,
3980: round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE,
3981: &pmap_kernel_l2ptp_kva, NULL);
1.1 matt 3982:
1.134 thorpej 3983: size = (size + (L2_BUCKET_SIZE - 1)) / L2_BUCKET_SIZE;
3984: pmap_alloc_specials(&virtual_avail,
3985: round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE,
3986: &pmap_kernel_l2dtable_kva, NULL);
1.1 matt 3987:
1.134 thorpej 3988: /*
3989: * init the static-global locks and global pmap list.
3990: */
3991: #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
3992: spinlockinit(&pmap_main_lock, "pmaplk", 0);
3993: #endif
1.1 matt 3994:
1.134 thorpej 3995: /*
3996: * We can now initialise the first L1's metadata.
3997: */
3998: SLIST_INIT(&l1_list);
3999: TAILQ_INIT(&l1_lru_list);
4000: simple_lock_init(&l1_lru_lock);
4001: pmap_init_l1(l1, kernel_l1pt);
1.1 matt 4002:
1.163.4.2! garbled 4003: /* Set up vector page L1 details, if necessary */
! 4004: if (vector_page < KERNEL_BASE) {
! 4005: pm->pm_pl1vec = &pm->pm_l1->l1_kva[L1_IDX(vector_page)];
! 4006: l2b = pmap_get_l2_bucket(pm, vector_page);
! 4007: pm->pm_l1vec = l2b->l2b_phys | L1_C_PROTO |
! 4008: L1_C_DOM(pm->pm_domain);
! 4009: } else
! 4010: pm->pm_pl1vec = NULL;
! 4011:
1.1 matt 4012: /*
1.134 thorpej 4013: * Initialize the pmap pool and cache
1.1 matt 4014: */
1.134 thorpej 4015: pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl",
1.162 ad 4016: &pool_allocator_nointr, IPL_NONE);
1.134 thorpej 4017: pool_cache_init(&pmap_pmap_cache, &pmap_pmap_pool,
4018: pmap_pmap_ctor, NULL, NULL);
4019: LIST_INIT(&pmap_pmaps);
4020: LIST_INSERT_HEAD(&pmap_pmaps, pm, pm_list);
1.1 matt 4021:
1.134 thorpej 4022: /*
4023: * Initialize the pv pool.
4024: */
4025: pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvepl",
1.162 ad 4026: &pmap_bootstrap_pv_allocator, IPL_NONE);
1.29 rearnsha 4027:
1.134 thorpej 4028: /*
4029: * Initialize the L2 dtable pool and cache.
4030: */
4031: pool_init(&pmap_l2dtable_pool, sizeof(struct l2_dtable), 0, 0, 0,
1.162 ad 4032: "l2dtblpl", NULL, IPL_NONE);
1.134 thorpej 4033: pool_cache_init(&pmap_l2dtable_cache, &pmap_l2dtable_pool,
4034: pmap_l2dtable_ctor, NULL, NULL);
1.1 matt 4035:
1.134 thorpej 4036: /*
4037: * Initialise the L2 descriptor table pool and cache
4038: */
4039: pool_init(&pmap_l2ptp_pool, L2_TABLE_SIZE_REAL, 0, L2_TABLE_SIZE_REAL,
1.162 ad 4040: 0, "l2ptppl", NULL, IPL_NONE);
1.134 thorpej 4041: pool_cache_init(&pmap_l2ptp_cache, &pmap_l2ptp_pool,
4042: pmap_l2ptp_ctor, NULL, NULL);
1.61 thorpej 4043:
1.134 thorpej 4044: cpu_dcache_wbinv_all();
1.1 matt 4045: }
4046:
1.134 thorpej 4047: static int
4048: pmap_set_pt_cache_mode(pd_entry_t *kl1, vaddr_t va)
1.1 matt 4049: {
1.134 thorpej 4050: pd_entry_t *pdep, pde;
4051: pt_entry_t *ptep, pte;
4052: vaddr_t pa;
4053: int rv = 0;
4054:
4055: /*
4056: * Make sure the descriptor itself has the correct cache mode
4057: */
4058: pdep = &kl1[L1_IDX(va)];
4059: pde = *pdep;
4060:
4061: if (l1pte_section_p(pde)) {
4062: if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) {
4063: *pdep = (pde & ~L1_S_CACHE_MASK) |
4064: pte_l1_s_cache_mode_pt;
4065: PTE_SYNC(pdep);
4066: cpu_dcache_wbinv_range((vaddr_t)pdep, sizeof(*pdep));
4067: rv = 1;
4068: }
4069: } else {
4070: pa = (paddr_t)(pde & L1_C_ADDR_MASK);
4071: ptep = (pt_entry_t *)kernel_pt_lookup(pa);
4072: if (ptep == NULL)
4073: panic("pmap_bootstrap: No L2 for L2 @ va %p\n", ptep);
4074:
4075: ptep = &ptep[l2pte_index(va)];
4076: pte = *ptep;
4077: if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) {
4078: *ptep = (pte & ~L2_S_CACHE_MASK) |
4079: pte_l2_s_cache_mode_pt;
4080: PTE_SYNC(ptep);
4081: cpu_dcache_wbinv_range((vaddr_t)ptep, sizeof(*ptep));
4082: rv = 1;
4083: }
4084: }
4085:
4086: return (rv);
4087: }
1.1 matt 4088:
1.134 thorpej 4089: static void
4090: pmap_alloc_specials(vaddr_t *availp, int pages, vaddr_t *vap, pt_entry_t **ptep)
4091: {
4092: vaddr_t va = *availp;
4093: struct l2_bucket *l2b;
1.1 matt 4094:
1.134 thorpej 4095: if (ptep) {
4096: l2b = pmap_get_l2_bucket(pmap_kernel(), va);
4097: if (l2b == NULL)
4098: panic("pmap_alloc_specials: no l2b for 0x%lx", va);
1.62 thorpej 4099:
1.134 thorpej 4100: if (ptep)
4101: *ptep = &l2b->l2b_kva[l2pte_index(va)];
1.1 matt 4102: }
4103:
1.134 thorpej 4104: *vap = va;
4105: *availp = va + (PAGE_SIZE * pages);
4106: }
4107:
4108: void
4109: pmap_init(void)
4110: {
4111: extern int physmem;
1.1 matt 4112:
1.113 thorpej 4113: /*
1.134 thorpej 4114: * Set the available memory vars - These do not map to real memory
4115: * addresses and cannot as the physical memory is fragmented.
4116: * They are used by ps for %mem calculations.
4117: * One could argue whether this should be the entire memory or just
4118: * the memory that is useable in a user process.
1.113 thorpej 4119: */
1.134 thorpej 4120: avail_start = 0;
4121: avail_end = physmem * PAGE_SIZE;
1.63 thorpej 4122:
1.1 matt 4123: /*
1.134 thorpej 4124: * Now we need to free enough pv_entry structures to allow us to get
4125: * the kmem_map/kmem_object allocated and inited (done after this
4126: * function is finished). to do this we allocate one bootstrap page out
4127: * of kernel_map and use it to provide an initial pool of pv_entry
4128: * structures. we never free this page.
1.1 matt 4129: */
1.134 thorpej 4130: pool_setlowat(&pmap_pv_pool,
4131: (PAGE_SIZE / sizeof(struct pv_entry)) * 2);
1.62 thorpej 4132:
1.160 thorpej 4133: pmap_initialized = true;
1.1 matt 4134: }
1.17 chris 4135:
1.134 thorpej 4136: static vaddr_t last_bootstrap_page = 0;
4137: static void *free_bootstrap_pages = NULL;
1.1 matt 4138:
1.134 thorpej 4139: static void *
4140: pmap_bootstrap_pv_page_alloc(struct pool *pp, int flags)
1.1 matt 4141: {
1.134 thorpej 4142: extern void *pool_page_alloc(struct pool *, int);
4143: vaddr_t new_page;
4144: void *rv;
4145:
4146: if (pmap_initialized)
4147: return (pool_page_alloc(pp, flags));
4148:
4149: if (free_bootstrap_pages) {
4150: rv = free_bootstrap_pages;
4151: free_bootstrap_pages = *((void **)rv);
4152: return (rv);
4153: }
4154:
1.151 yamt 4155: new_page = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
4156: UVM_KMF_WIRED | ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT));
1.1 matt 4157:
1.134 thorpej 4158: KASSERT(new_page > last_bootstrap_page);
4159: last_bootstrap_page = new_page;
4160: return ((void *)new_page);
1.17 chris 4161: }
4162:
1.134 thorpej 4163: static void
4164: pmap_bootstrap_pv_page_free(struct pool *pp, void *v)
1.17 chris 4165: {
1.134 thorpej 4166: extern void pool_page_free(struct pool *, void *);
1.17 chris 4167:
1.150 joff 4168: if ((vaddr_t)v <= last_bootstrap_page) {
4169: *((void **)v) = free_bootstrap_pages;
4170: free_bootstrap_pages = v;
1.134 thorpej 4171: return;
4172: }
1.114 thorpej 4173:
1.150 joff 4174: if (pmap_initialized) {
4175: pool_page_free(pp, v);
1.134 thorpej 4176: return;
1.57 thorpej 4177: }
1.17 chris 4178: }
4179:
4180: /*
1.134 thorpej 4181: * pmap_postinit()
1.17 chris 4182: *
1.134 thorpej 4183: * This routine is called after the vm and kmem subsystems have been
4184: * initialised. This allows the pmap code to perform any initialisation
4185: * that can only be done one the memory allocation is in place.
1.17 chris 4186: */
1.134 thorpej 4187: void
4188: pmap_postinit(void)
1.17 chris 4189: {
1.134 thorpej 4190: extern paddr_t physical_start, physical_end;
4191: struct l2_bucket *l2b;
4192: struct l1_ttable *l1;
4193: struct pglist plist;
4194: struct vm_page *m;
4195: pd_entry_t *pl1pt;
4196: pt_entry_t *ptep, pte;
4197: vaddr_t va, eva;
4198: u_int loop, needed;
4199: int error;
1.114 thorpej 4200:
1.134 thorpej 4201: pool_setlowat(&pmap_l2ptp_pool,
4202: (PAGE_SIZE / L2_TABLE_SIZE_REAL) * 4);
4203: pool_setlowat(&pmap_l2dtable_pool,
4204: (PAGE_SIZE / sizeof(struct l2_dtable)) * 2);
1.17 chris 4205:
1.134 thorpej 4206: needed = (maxproc / PMAP_DOMAINS) + ((maxproc % PMAP_DOMAINS) ? 1 : 0);
4207: needed -= 1;
1.48 chris 4208:
1.134 thorpej 4209: l1 = malloc(sizeof(*l1) * needed, M_VMPMAP, M_WAITOK);
1.48 chris 4210:
1.134 thorpej 4211: for (loop = 0; loop < needed; loop++, l1++) {
4212: /* Allocate a L1 page table */
1.151 yamt 4213: va = uvm_km_alloc(kernel_map, L1_TABLE_SIZE, 0, UVM_KMF_VAONLY);
1.134 thorpej 4214: if (va == 0)
4215: panic("Cannot allocate L1 KVM");
4216:
4217: error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start,
4218: physical_end, L1_TABLE_SIZE, 0, &plist, 1, M_WAITOK);
4219: if (error)
4220: panic("Cannot allocate L1 physical pages");
4221:
4222: m = TAILQ_FIRST(&plist);
4223: eva = va + L1_TABLE_SIZE;
4224: pl1pt = (pd_entry_t *)va;
1.48 chris 4225:
1.134 thorpej 4226: while (m && va < eva) {
4227: paddr_t pa = VM_PAGE_TO_PHYS(m);
1.48 chris 4228:
1.134 thorpej 4229: pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE);
1.48 chris 4230:
4231: /*
1.134 thorpej 4232: * Make sure the L1 descriptor table is mapped
4233: * with the cache-mode set to write-through.
1.48 chris 4234: */
1.134 thorpej 4235: l2b = pmap_get_l2_bucket(pmap_kernel(), va);
4236: ptep = &l2b->l2b_kva[l2pte_index(va)];
4237: pte = *ptep;
4238: pte = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt;
4239: *ptep = pte;
4240: PTE_SYNC(ptep);
4241: cpu_tlb_flushD_SE(va);
1.48 chris 4242:
1.134 thorpej 4243: va += PAGE_SIZE;
1.149 chris 4244: m = TAILQ_NEXT(m, pageq);
1.48 chris 4245: }
4246:
1.134 thorpej 4247: #ifdef DIAGNOSTIC
4248: if (m)
4249: panic("pmap_alloc_l1pt: pglist not empty");
4250: #endif /* DIAGNOSTIC */
1.48 chris 4251:
1.134 thorpej 4252: pmap_init_l1(l1, pl1pt);
1.48 chris 4253: }
4254:
1.134 thorpej 4255: #ifdef DEBUG
4256: printf("pmap_postinit: Allocated %d static L1 descriptor tables\n",
4257: needed);
4258: #endif
1.48 chris 4259: }
4260:
1.76 thorpej 4261: /*
1.134 thorpej 4262: * Note that the following routines are used by board-specific initialisation
4263: * code to configure the initial kernel page tables.
4264: *
4265: * If ARM32_NEW_VM_LAYOUT is *not* defined, they operate on the assumption that
4266: * L2 page-table pages are 4KB in size and use 4 L1 slots. This mimics the
4267: * behaviour of the old pmap, and provides an easy migration path for
4268: * initial bring-up of the new pmap on existing ports. Fortunately,
4269: * pmap_bootstrap() compensates for this hackery. This is only a stop-gap and
4270: * will be deprecated.
1.76 thorpej 4271: *
1.134 thorpej 4272: * If ARM32_NEW_VM_LAYOUT *is* defined, these functions deal with 1KB L2 page
4273: * tables.
1.76 thorpej 4274: */
1.40 thorpej 4275:
4276: /*
1.46 thorpej 4277: * This list exists for the benefit of pmap_map_chunk(). It keeps track
4278: * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can
4279: * find them as necessary.
4280: *
1.134 thorpej 4281: * Note that the data on this list MUST remain valid after initarm() returns,
4282: * as pmap_bootstrap() uses it to contruct L2 table metadata.
1.46 thorpej 4283: */
4284: SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list);
4285:
4286: static vaddr_t
4287: kernel_pt_lookup(paddr_t pa)
4288: {
4289: pv_addr_t *pv;
4290:
4291: SLIST_FOREACH(pv, &kernel_pt_list, pv_list) {
1.134 thorpej 4292: #ifndef ARM32_NEW_VM_LAYOUT
4293: if (pv->pv_pa == (pa & ~PGOFSET))
4294: return (pv->pv_va | (pa & PGOFSET));
4295: #else
1.46 thorpej 4296: if (pv->pv_pa == pa)
4297: return (pv->pv_va);
1.134 thorpej 4298: #endif
1.46 thorpej 4299: }
4300: return (0);
4301: }
4302:
4303: /*
1.40 thorpej 4304: * pmap_map_section:
4305: *
4306: * Create a single section mapping.
4307: */
4308: void
4309: pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache)
4310: {
4311: pd_entry_t *pde = (pd_entry_t *) l1pt;
1.134 thorpej 4312: pd_entry_t fl;
1.40 thorpej 4313:
1.81 thorpej 4314: KASSERT(((va | pa) & L1_S_OFFSET) == 0);
1.40 thorpej 4315:
1.134 thorpej 4316: switch (cache) {
4317: case PTE_NOCACHE:
4318: default:
4319: fl = 0;
4320: break;
4321:
4322: case PTE_CACHE:
4323: fl = pte_l1_s_cache_mode;
4324: break;
4325:
4326: case PTE_PAGETABLE:
4327: fl = pte_l1_s_cache_mode_pt;
4328: break;
4329: }
4330:
1.83 thorpej 4331: pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa |
1.134 thorpej 4332: L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL);
4333: PTE_SYNC(&pde[va >> L1_S_SHIFT]);
1.41 thorpej 4334: }
4335:
4336: /*
4337: * pmap_map_entry:
4338: *
4339: * Create a single page mapping.
4340: */
4341: void
1.47 thorpej 4342: pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache)
1.41 thorpej 4343: {
1.47 thorpej 4344: pd_entry_t *pde = (pd_entry_t *) l1pt;
1.134 thorpej 4345: pt_entry_t fl;
1.47 thorpej 4346: pt_entry_t *pte;
1.41 thorpej 4347:
4348: KASSERT(((va | pa) & PGOFSET) == 0);
4349:
1.134 thorpej 4350: switch (cache) {
4351: case PTE_NOCACHE:
4352: default:
4353: fl = 0;
4354: break;
4355:
4356: case PTE_CACHE:
4357: fl = pte_l2_s_cache_mode;
4358: break;
4359:
4360: case PTE_PAGETABLE:
4361: fl = pte_l2_s_cache_mode_pt;
4362: break;
4363: }
4364:
1.81 thorpej 4365: if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C)
1.47 thorpej 4366: panic("pmap_map_entry: no L2 table for VA 0x%08lx", va);
4367:
1.134 thorpej 4368: #ifndef ARM32_NEW_VM_LAYOUT
1.47 thorpej 4369: pte = (pt_entry_t *)
1.81 thorpej 4370: kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME);
1.134 thorpej 4371: #else
4372: pte = (pt_entry_t *) kernel_pt_lookup(pde[L1_IDX(va)] & L1_C_ADDR_MASK);
4373: #endif
1.47 thorpej 4374: if (pte == NULL)
4375: panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va);
4376:
1.134 thorpej 4377: #ifndef ARM32_NEW_VM_LAYOUT
4378: pte[(va >> PGSHIFT) & 0x3ff] =
4379: L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | fl;
4380: PTE_SYNC(&pte[(va >> PGSHIFT) & 0x3ff]);
4381: #else
4382: pte[l2pte_index(va)] =
4383: L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | fl;
4384: PTE_SYNC(&pte[l2pte_index(va)]);
4385: #endif
1.42 thorpej 4386: }
4387:
4388: /*
4389: * pmap_link_l2pt:
4390: *
1.134 thorpej 4391: * Link the L2 page table specified by "l2pv" into the L1
1.42 thorpej 4392: * page table at the slot for "va".
4393: */
4394: void
1.46 thorpej 4395: pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv)
1.42 thorpej 4396: {
1.134 thorpej 4397: pd_entry_t *pde = (pd_entry_t *) l1pt, proto;
1.81 thorpej 4398: u_int slot = va >> L1_S_SHIFT;
1.42 thorpej 4399:
1.134 thorpej 4400: #ifndef ARM32_NEW_VM_LAYOUT
4401: KASSERT((va & ((L1_S_SIZE * 4) - 1)) == 0);
1.46 thorpej 4402: KASSERT((l2pv->pv_pa & PGOFSET) == 0);
1.134 thorpej 4403: #endif
1.46 thorpej 4404:
1.134 thorpej 4405: proto = L1_S_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO;
4406:
4407: pde[slot + 0] = proto | (l2pv->pv_pa + 0x000);
4408: #ifdef ARM32_NEW_VM_LAYOUT
4409: PTE_SYNC(&pde[slot]);
4410: #else
4411: pde[slot + 1] = proto | (l2pv->pv_pa + 0x400);
4412: pde[slot + 2] = proto | (l2pv->pv_pa + 0x800);
4413: pde[slot + 3] = proto | (l2pv->pv_pa + 0xc00);
4414: PTE_SYNC_RANGE(&pde[slot + 0], 4);
4415: #endif
1.42 thorpej 4416:
1.46 thorpej 4417: SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list);
1.43 thorpej 4418: }
4419:
4420: /*
4421: * pmap_map_chunk:
4422: *
4423: * Map a chunk of memory using the most efficient mappings
4424: * possible (section, large page, small page) into the
4425: * provided L1 and L2 tables at the specified virtual address.
4426: */
4427: vsize_t
1.46 thorpej 4428: pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size,
4429: int prot, int cache)
1.43 thorpej 4430: {
4431: pd_entry_t *pde = (pd_entry_t *) l1pt;
1.134 thorpej 4432: pt_entry_t *pte, f1, f2s, f2l;
1.43 thorpej 4433: vsize_t resid;
1.134 thorpej 4434: int i;
1.43 thorpej 4435:
1.130 thorpej 4436: resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1);
1.43 thorpej 4437:
1.44 thorpej 4438: if (l1pt == 0)
4439: panic("pmap_map_chunk: no L1 table provided");
4440:
1.43 thorpej 4441: #ifdef VERBOSE_INIT_ARM
4442: printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx "
4443: "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache);
4444: #endif
4445:
1.134 thorpej 4446: switch (cache) {
4447: case PTE_NOCACHE:
4448: default:
4449: f1 = 0;
4450: f2l = 0;
4451: f2s = 0;
4452: break;
4453:
4454: case PTE_CACHE:
4455: f1 = pte_l1_s_cache_mode;
4456: f2l = pte_l2_l_cache_mode;
4457: f2s = pte_l2_s_cache_mode;
4458: break;
4459:
4460: case PTE_PAGETABLE:
4461: f1 = pte_l1_s_cache_mode_pt;
4462: f2l = pte_l2_l_cache_mode_pt;
4463: f2s = pte_l2_s_cache_mode_pt;
4464: break;
4465: }
4466:
1.43 thorpej 4467: size = resid;
4468:
4469: while (resid > 0) {
4470: /* See if we can use a section mapping. */
1.134 thorpej 4471: if (L1_S_MAPPABLE_P(va, pa, resid)) {
1.43 thorpej 4472: #ifdef VERBOSE_INIT_ARM
4473: printf("S");
4474: #endif
1.83 thorpej 4475: pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa |
1.134 thorpej 4476: L1_S_PROT(PTE_KERNEL, prot) | f1 |
4477: L1_S_DOM(PMAP_DOMAIN_KERNEL);
4478: PTE_SYNC(&pde[va >> L1_S_SHIFT]);
1.81 thorpej 4479: va += L1_S_SIZE;
4480: pa += L1_S_SIZE;
4481: resid -= L1_S_SIZE;
1.43 thorpej 4482: continue;
4483: }
1.45 thorpej 4484:
4485: /*
4486: * Ok, we're going to use an L2 table. Make sure
4487: * one is actually in the corresponding L1 slot
4488: * for the current VA.
4489: */
1.81 thorpej 4490: if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C)
1.46 thorpej 4491: panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va);
4492:
1.134 thorpej 4493: #ifndef ARM32_NEW_VM_LAYOUT
1.46 thorpej 4494: pte = (pt_entry_t *)
1.81 thorpej 4495: kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME);
1.134 thorpej 4496: #else
4497: pte = (pt_entry_t *) kernel_pt_lookup(
4498: pde[L1_IDX(va)] & L1_C_ADDR_MASK);
4499: #endif
1.46 thorpej 4500: if (pte == NULL)
4501: panic("pmap_map_chunk: can't find L2 table for VA"
4502: "0x%08lx", va);
1.43 thorpej 4503:
4504: /* See if we can use a L2 large page mapping. */
1.134 thorpej 4505: if (L2_L_MAPPABLE_P(va, pa, resid)) {
1.43 thorpej 4506: #ifdef VERBOSE_INIT_ARM
4507: printf("L");
4508: #endif
4509: for (i = 0; i < 16; i++) {
1.134 thorpej 4510: #ifndef ARM32_NEW_VM_LAYOUT
1.43 thorpej 4511: pte[((va >> PGSHIFT) & 0x3f0) + i] =
1.83 thorpej 4512: L2_L_PROTO | pa |
1.134 thorpej 4513: L2_L_PROT(PTE_KERNEL, prot) | f2l;
4514: PTE_SYNC(&pte[((va >> PGSHIFT) & 0x3f0) + i]);
4515: #else
4516: pte[l2pte_index(va) + i] =
4517: L2_L_PROTO | pa |
4518: L2_L_PROT(PTE_KERNEL, prot) | f2l;
4519: PTE_SYNC(&pte[l2pte_index(va) + i]);
4520: #endif
1.43 thorpej 4521: }
1.81 thorpej 4522: va += L2_L_SIZE;
4523: pa += L2_L_SIZE;
4524: resid -= L2_L_SIZE;
1.43 thorpej 4525: continue;
4526: }
4527:
4528: /* Use a small page mapping. */
4529: #ifdef VERBOSE_INIT_ARM
4530: printf("P");
4531: #endif
1.134 thorpej 4532: #ifndef ARM32_NEW_VM_LAYOUT
4533: pte[(va >> PGSHIFT) & 0x3ff] =
4534: L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | f2s;
4535: PTE_SYNC(&pte[(va >> PGSHIFT) & 0x3ff]);
4536: #else
4537: pte[l2pte_index(va)] =
4538: L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | f2s;
4539: PTE_SYNC(&pte[l2pte_index(va)]);
4540: #endif
1.130 thorpej 4541: va += PAGE_SIZE;
4542: pa += PAGE_SIZE;
4543: resid -= PAGE_SIZE;
1.43 thorpej 4544: }
4545: #ifdef VERBOSE_INIT_ARM
4546: printf("\n");
4547: #endif
4548: return (size);
1.135 thorpej 4549: }
4550:
4551: /********************** Static device map routines ***************************/
4552:
4553: static const struct pmap_devmap *pmap_devmap_table;
4554:
4555: /*
1.136 thorpej 4556: * Register the devmap table. This is provided in case early console
4557: * initialization needs to register mappings created by bootstrap code
4558: * before pmap_devmap_bootstrap() is called.
4559: */
4560: void
4561: pmap_devmap_register(const struct pmap_devmap *table)
4562: {
4563:
4564: pmap_devmap_table = table;
4565: }
4566:
4567: /*
1.135 thorpej 4568: * Map all of the static regions in the devmap table, and remember
4569: * the devmap table so other parts of the kernel can look up entries
4570: * later.
4571: */
4572: void
4573: pmap_devmap_bootstrap(vaddr_t l1pt, const struct pmap_devmap *table)
4574: {
4575: int i;
4576:
4577: pmap_devmap_table = table;
4578:
4579: for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) {
4580: #ifdef VERBOSE_INIT_ARM
4581: printf("devmap: %08lx -> %08lx @ %08lx\n",
4582: pmap_devmap_table[i].pd_pa,
4583: pmap_devmap_table[i].pd_pa +
4584: pmap_devmap_table[i].pd_size - 1,
4585: pmap_devmap_table[i].pd_va);
4586: #endif
4587: pmap_map_chunk(l1pt, pmap_devmap_table[i].pd_va,
4588: pmap_devmap_table[i].pd_pa,
4589: pmap_devmap_table[i].pd_size,
4590: pmap_devmap_table[i].pd_prot,
4591: pmap_devmap_table[i].pd_cache);
4592: }
4593: }
4594:
4595: const struct pmap_devmap *
4596: pmap_devmap_find_pa(paddr_t pa, psize_t size)
4597: {
1.153 scw 4598: uint64_t endpa;
1.135 thorpej 4599: int i;
4600:
4601: if (pmap_devmap_table == NULL)
4602: return (NULL);
4603:
1.158 christos 4604: endpa = (uint64_t)pa + (uint64_t)(size - 1);
1.153 scw 4605:
1.135 thorpej 4606: for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) {
4607: if (pa >= pmap_devmap_table[i].pd_pa &&
1.153 scw 4608: endpa <= (uint64_t)pmap_devmap_table[i].pd_pa +
1.158 christos 4609: (uint64_t)(pmap_devmap_table[i].pd_size - 1))
1.135 thorpej 4610: return (&pmap_devmap_table[i]);
4611: }
4612:
4613: return (NULL);
4614: }
4615:
4616: const struct pmap_devmap *
4617: pmap_devmap_find_va(vaddr_t va, vsize_t size)
4618: {
4619: int i;
4620:
4621: if (pmap_devmap_table == NULL)
4622: return (NULL);
4623:
4624: for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) {
4625: if (va >= pmap_devmap_table[i].pd_va &&
1.158 christos 4626: va + size - 1 <= pmap_devmap_table[i].pd_va +
4627: pmap_devmap_table[i].pd_size - 1)
1.135 thorpej 4628: return (&pmap_devmap_table[i]);
4629: }
4630:
4631: return (NULL);
1.40 thorpej 4632: }
1.85 thorpej 4633:
4634: /********************** PTE initialization routines **************************/
4635:
4636: /*
4637: * These routines are called when the CPU type is identified to set up
4638: * the PTE prototypes, cache modes, etc.
4639: *
4640: * The variables are always here, just in case LKMs need to reference
4641: * them (though, they shouldn't).
4642: */
4643:
1.86 thorpej 4644: pt_entry_t pte_l1_s_cache_mode;
1.134 thorpej 4645: pt_entry_t pte_l1_s_cache_mode_pt;
1.86 thorpej 4646: pt_entry_t pte_l1_s_cache_mask;
4647:
4648: pt_entry_t pte_l2_l_cache_mode;
1.134 thorpej 4649: pt_entry_t pte_l2_l_cache_mode_pt;
1.86 thorpej 4650: pt_entry_t pte_l2_l_cache_mask;
4651:
4652: pt_entry_t pte_l2_s_cache_mode;
1.134 thorpej 4653: pt_entry_t pte_l2_s_cache_mode_pt;
1.86 thorpej 4654: pt_entry_t pte_l2_s_cache_mask;
1.85 thorpej 4655:
4656: pt_entry_t pte_l2_s_prot_u;
4657: pt_entry_t pte_l2_s_prot_w;
4658: pt_entry_t pte_l2_s_prot_mask;
4659:
4660: pt_entry_t pte_l1_s_proto;
4661: pt_entry_t pte_l1_c_proto;
4662: pt_entry_t pte_l2_s_proto;
4663:
1.88 thorpej 4664: void (*pmap_copy_page_func)(paddr_t, paddr_t);
4665: void (*pmap_zero_page_func)(paddr_t);
4666:
1.131 thorpej 4667: #if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0
1.85 thorpej 4668: void
4669: pmap_pte_init_generic(void)
4670: {
4671:
1.86 thorpej 4672: pte_l1_s_cache_mode = L1_S_B|L1_S_C;
4673: pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic;
4674:
4675: pte_l2_l_cache_mode = L2_B|L2_C;
4676: pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic;
4677:
4678: pte_l2_s_cache_mode = L2_B|L2_C;
4679: pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic;
1.85 thorpej 4680:
1.134 thorpej 4681: /*
4682: * If we have a write-through cache, set B and C. If
4683: * we have a write-back cache, then we assume setting
4684: * only C will make those pages write-through.
4685: */
4686: if (cpufuncs.cf_dcache_wb_range == (void *) cpufunc_nullop) {
4687: pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C;
4688: pte_l2_l_cache_mode_pt = L2_B|L2_C;
4689: pte_l2_s_cache_mode_pt = L2_B|L2_C;
4690: } else {
4691: pte_l1_s_cache_mode_pt = L1_S_C;
4692: pte_l2_l_cache_mode_pt = L2_C;
4693: pte_l2_s_cache_mode_pt = L2_C;
4694: }
4695:
1.85 thorpej 4696: pte_l2_s_prot_u = L2_S_PROT_U_generic;
4697: pte_l2_s_prot_w = L2_S_PROT_W_generic;
4698: pte_l2_s_prot_mask = L2_S_PROT_MASK_generic;
4699:
4700: pte_l1_s_proto = L1_S_PROTO_generic;
4701: pte_l1_c_proto = L1_C_PROTO_generic;
4702: pte_l2_s_proto = L2_S_PROTO_generic;
1.88 thorpej 4703:
4704: pmap_copy_page_func = pmap_copy_page_generic;
4705: pmap_zero_page_func = pmap_zero_page_generic;
1.85 thorpej 4706: }
4707:
1.131 thorpej 4708: #if defined(CPU_ARM8)
4709: void
4710: pmap_pte_init_arm8(void)
4711: {
4712:
1.134 thorpej 4713: /*
4714: * ARM8 is compatible with generic, but we need to use
4715: * the page tables uncached.
4716: */
1.131 thorpej 4717: pmap_pte_init_generic();
1.134 thorpej 4718:
4719: pte_l1_s_cache_mode_pt = 0;
4720: pte_l2_l_cache_mode_pt = 0;
4721: pte_l2_s_cache_mode_pt = 0;
1.131 thorpej 4722: }
4723: #endif /* CPU_ARM8 */
4724:
1.148 bsh 4725: #if defined(CPU_ARM9) && defined(ARM9_CACHE_WRITE_THROUGH)
1.85 thorpej 4726: void
4727: pmap_pte_init_arm9(void)
4728: {
4729:
4730: /*
4731: * ARM9 is compatible with generic, but we want to use
4732: * write-through caching for now.
4733: */
4734: pmap_pte_init_generic();
1.86 thorpej 4735:
4736: pte_l1_s_cache_mode = L1_S_C;
4737: pte_l2_l_cache_mode = L2_C;
4738: pte_l2_s_cache_mode = L2_C;
1.134 thorpej 4739:
4740: pte_l1_s_cache_mode_pt = L1_S_C;
4741: pte_l2_l_cache_mode_pt = L2_C;
4742: pte_l2_s_cache_mode_pt = L2_C;
1.85 thorpej 4743: }
4744: #endif /* CPU_ARM9 */
1.134 thorpej 4745: #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */
1.138 rearnsha 4746:
4747: #if defined(CPU_ARM10)
4748: void
4749: pmap_pte_init_arm10(void)
4750: {
4751:
4752: /*
4753: * ARM10 is compatible with generic, but we want to use
4754: * write-through caching for now.
4755: */
4756: pmap_pte_init_generic();
4757:
4758: pte_l1_s_cache_mode = L1_S_B | L1_S_C;
4759: pte_l2_l_cache_mode = L2_B | L2_C;
4760: pte_l2_s_cache_mode = L2_B | L2_C;
4761:
4762: pte_l1_s_cache_mode_pt = L1_S_C;
4763: pte_l2_l_cache_mode_pt = L2_C;
4764: pte_l2_s_cache_mode_pt = L2_C;
4765:
4766: }
4767: #endif /* CPU_ARM10 */
1.131 thorpej 4768:
4769: #if ARM_MMU_SA1 == 1
4770: void
4771: pmap_pte_init_sa1(void)
4772: {
4773:
1.134 thorpej 4774: /*
4775: * The StrongARM SA-1 cache does not have a write-through
4776: * mode. So, do the generic initialization, then reset
4777: * the page table cache mode to B=1,C=1, and note that
4778: * the PTEs need to be sync'd.
4779: */
1.131 thorpej 4780: pmap_pte_init_generic();
1.134 thorpej 4781:
4782: pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C;
4783: pte_l2_l_cache_mode_pt = L2_B|L2_C;
4784: pte_l2_s_cache_mode_pt = L2_B|L2_C;
4785:
4786: pmap_needs_pte_sync = 1;
1.131 thorpej 4787: }
1.134 thorpej 4788: #endif /* ARM_MMU_SA1 == 1*/
1.85 thorpej 4789:
4790: #if ARM_MMU_XSCALE == 1
1.141 scw 4791: #if (ARM_NMMUS > 1)
4792: static u_int xscale_use_minidata;
4793: #endif
4794:
1.85 thorpej 4795: void
4796: pmap_pte_init_xscale(void)
4797: {
1.96 thorpej 4798: uint32_t auxctl;
1.134 thorpej 4799: int write_through = 0;
1.85 thorpej 4800:
1.96 thorpej 4801: pte_l1_s_cache_mode = L1_S_B|L1_S_C;
1.86 thorpej 4802: pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale;
4803:
1.96 thorpej 4804: pte_l2_l_cache_mode = L2_B|L2_C;
1.86 thorpej 4805: pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale;
4806:
1.96 thorpej 4807: pte_l2_s_cache_mode = L2_B|L2_C;
1.86 thorpej 4808: pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale;
1.106 thorpej 4809:
1.134 thorpej 4810: pte_l1_s_cache_mode_pt = L1_S_C;
4811: pte_l2_l_cache_mode_pt = L2_C;
4812: pte_l2_s_cache_mode_pt = L2_C;
4813:
1.106 thorpej 4814: #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE
4815: /*
4816: * The XScale core has an enhanced mode where writes that
4817: * miss the cache cause a cache line to be allocated. This
4818: * is significantly faster than the traditional, write-through
4819: * behavior of this case.
4820: */
4821: pte_l1_s_cache_mode |= L1_S_XSCALE_TEX(TEX_XSCALE_X);
4822: pte_l2_l_cache_mode |= L2_XSCALE_L_TEX(TEX_XSCALE_X);
4823: pte_l2_s_cache_mode |= L2_XSCALE_T_TEX(TEX_XSCALE_X);
4824: #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */
1.85 thorpej 4825:
1.95 thorpej 4826: #ifdef XSCALE_CACHE_WRITE_THROUGH
4827: /*
4828: * Some versions of the XScale core have various bugs in
4829: * their cache units, the work-around for which is to run
4830: * the cache in write-through mode. Unfortunately, this
4831: * has a major (negative) impact on performance. So, we
4832: * go ahead and run fast-and-loose, in the hopes that we
4833: * don't line up the planets in a way that will trip the
4834: * bugs.
4835: *
4836: * However, we give you the option to be slow-but-correct.
4837: */
1.129 bsh 4838: write_through = 1;
4839: #elif defined(XSCALE_CACHE_WRITE_BACK)
1.134 thorpej 4840: /* force write back cache mode */
1.129 bsh 4841: write_through = 0;
1.154 bsh 4842: #elif defined(CPU_XSCALE_PXA250) || defined(CPU_XSCALE_PXA270)
1.129 bsh 4843: /*
4844: * Intel PXA2[15]0 processors are known to have a bug in
4845: * write-back cache on revision 4 and earlier (stepping
4846: * A[01] and B[012]). Fixed for C0 and later.
4847: */
4848: {
1.134 thorpej 4849: uint32_t id, type;
1.129 bsh 4850:
4851: id = cpufunc_id();
4852: type = id & ~(CPU_ID_XSCALE_COREREV_MASK|CPU_ID_REVISION_MASK);
4853:
4854: if (type == CPU_ID_PXA250 || type == CPU_ID_PXA210) {
4855: if ((id & CPU_ID_REVISION_MASK) < 5) {
4856: /* write through for stepping A0-1 and B0-2 */
4857: write_through = 1;
4858: }
4859: }
4860: }
1.95 thorpej 4861: #endif /* XSCALE_CACHE_WRITE_THROUGH */
1.129 bsh 4862:
4863: if (write_through) {
4864: pte_l1_s_cache_mode = L1_S_C;
4865: pte_l2_l_cache_mode = L2_C;
4866: pte_l2_s_cache_mode = L2_C;
4867: }
1.95 thorpej 4868:
1.141 scw 4869: #if (ARM_NMMUS > 1)
4870: xscale_use_minidata = 1;
4871: #endif
4872:
1.85 thorpej 4873: pte_l2_s_prot_u = L2_S_PROT_U_xscale;
4874: pte_l2_s_prot_w = L2_S_PROT_W_xscale;
4875: pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale;
4876:
4877: pte_l1_s_proto = L1_S_PROTO_xscale;
4878: pte_l1_c_proto = L1_C_PROTO_xscale;
4879: pte_l2_s_proto = L2_S_PROTO_xscale;
1.88 thorpej 4880:
4881: pmap_copy_page_func = pmap_copy_page_xscale;
4882: pmap_zero_page_func = pmap_zero_page_xscale;
1.96 thorpej 4883:
4884: /*
4885: * Disable ECC protection of page table access, for now.
4886: */
1.157 perry 4887: __asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl));
1.96 thorpej 4888: auxctl &= ~XSCALE_AUXCTL_P;
1.157 perry 4889: __asm volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl));
1.85 thorpej 4890: }
1.87 thorpej 4891:
4892: /*
4893: * xscale_setup_minidata:
4894: *
4895: * Set up the mini-data cache clean area. We require the
4896: * caller to allocate the right amount of physically and
4897: * virtually contiguous space.
4898: */
4899: void
4900: xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa)
4901: {
4902: extern vaddr_t xscale_minidata_clean_addr;
4903: extern vsize_t xscale_minidata_clean_size; /* already initialized */
4904: pd_entry_t *pde = (pd_entry_t *) l1pt;
4905: pt_entry_t *pte;
4906: vsize_t size;
1.96 thorpej 4907: uint32_t auxctl;
1.87 thorpej 4908:
4909: xscale_minidata_clean_addr = va;
4910:
4911: /* Round it to page size. */
4912: size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME;
4913:
4914: for (; size != 0;
4915: va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) {
1.134 thorpej 4916: #ifndef ARM32_NEW_VM_LAYOUT
1.87 thorpej 4917: pte = (pt_entry_t *)
4918: kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME);
1.134 thorpej 4919: #else
4920: pte = (pt_entry_t *) kernel_pt_lookup(
4921: pde[L1_IDX(va)] & L1_C_ADDR_MASK);
4922: #endif
1.87 thorpej 4923: if (pte == NULL)
4924: panic("xscale_setup_minidata: can't find L2 table for "
4925: "VA 0x%08lx", va);
1.134 thorpej 4926: #ifndef ARM32_NEW_VM_LAYOUT
4927: pte[(va >> PGSHIFT) & 0x3ff] =
4928: #else
4929: pte[l2pte_index(va)] =
4930: #endif
4931: L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) |
1.87 thorpej 4932: L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X);
4933: }
1.96 thorpej 4934:
4935: /*
4936: * Configure the mini-data cache for write-back with
4937: * read/write-allocate.
4938: *
4939: * NOTE: In order to reconfigure the mini-data cache, we must
4940: * make sure it contains no valid data! In order to do that,
4941: * we must issue a global data cache invalidate command!
4942: *
4943: * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED!
4944: * THIS IS VERY IMPORTANT!
4945: */
1.134 thorpej 4946:
1.96 thorpej 4947: /* Invalidate data and mini-data. */
1.157 perry 4948: __asm volatile("mcr p15, 0, %0, c7, c6, 0" : : "r" (0));
4949: __asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl));
1.96 thorpej 4950: auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA;
1.157 perry 4951: __asm volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl));
1.87 thorpej 4952: }
1.141 scw 4953:
4954: /*
4955: * Change the PTEs for the specified kernel mappings such that they
4956: * will use the mini data cache instead of the main data cache.
4957: */
4958: void
4959: pmap_uarea(vaddr_t va)
4960: {
4961: struct l2_bucket *l2b;
4962: pt_entry_t *ptep, *sptep, pte;
4963: vaddr_t next_bucket, eva;
4964:
4965: #if (ARM_NMMUS > 1)
4966: if (xscale_use_minidata == 0)
4967: return;
4968: #endif
4969:
4970: eva = va + USPACE;
4971:
4972: while (va < eva) {
4973: next_bucket = L2_NEXT_BUCKET(va);
4974: if (next_bucket > eva)
4975: next_bucket = eva;
4976:
4977: l2b = pmap_get_l2_bucket(pmap_kernel(), va);
4978: KDASSERT(l2b != NULL);
4979:
4980: sptep = ptep = &l2b->l2b_kva[l2pte_index(va)];
4981:
4982: while (va < next_bucket) {
4983: pte = *ptep;
4984: if (!l2pte_minidata(pte)) {
4985: cpu_dcache_wbinv_range(va, PAGE_SIZE);
4986: cpu_tlb_flushD_SE(va);
4987: *ptep = pte & ~L2_B;
4988: }
4989: ptep++;
4990: va += PAGE_SIZE;
4991: }
4992: PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep));
4993: }
4994: cpu_cpwait();
4995: }
1.85 thorpej 4996: #endif /* ARM_MMU_XSCALE == 1 */
1.134 thorpej 4997:
4998: #if defined(DDB)
4999: /*
5000: * A couple of ddb-callable functions for dumping pmaps
5001: */
5002: void pmap_dump_all(void);
5003: void pmap_dump(pmap_t);
5004:
5005: void
5006: pmap_dump_all(void)
5007: {
5008: pmap_t pm;
5009:
5010: LIST_FOREACH(pm, &pmap_pmaps, pm_list) {
5011: if (pm == pmap_kernel())
5012: continue;
5013: pmap_dump(pm);
5014: printf("\n");
5015: }
5016: }
5017:
5018: static pt_entry_t ncptes[64];
5019: static void pmap_dump_ncpg(pmap_t);
5020:
5021: void
5022: pmap_dump(pmap_t pm)
5023: {
5024: struct l2_dtable *l2;
5025: struct l2_bucket *l2b;
5026: pt_entry_t *ptep, pte;
5027: vaddr_t l2_va, l2b_va, va;
5028: int i, j, k, occ, rows = 0;
5029:
5030: if (pm == pmap_kernel())
5031: printf("pmap_kernel (%p): ", pm);
5032: else
5033: printf("user pmap (%p): ", pm);
5034:
5035: printf("domain %d, l1 at %p\n", pm->pm_domain, pm->pm_l1->l1_kva);
5036:
5037: l2_va = 0;
5038: for (i = 0; i < L2_SIZE; i++, l2_va += 0x01000000) {
5039: l2 = pm->pm_l2[i];
5040:
5041: if (l2 == NULL || l2->l2_occupancy == 0)
5042: continue;
5043:
5044: l2b_va = l2_va;
5045: for (j = 0; j < L2_BUCKET_SIZE; j++, l2b_va += 0x00100000) {
5046: l2b = &l2->l2_bucket[j];
5047:
5048: if (l2b->l2b_occupancy == 0 || l2b->l2b_kva == NULL)
5049: continue;
5050:
5051: ptep = l2b->l2b_kva;
5052:
5053: for (k = 0; k < 256 && ptep[k] == 0; k++)
5054: ;
5055:
5056: k &= ~63;
5057: occ = l2b->l2b_occupancy;
5058: va = l2b_va + (k * 4096);
5059: for (; k < 256; k++, va += 0x1000) {
1.142 chris 5060: char ch = ' ';
1.134 thorpej 5061: if ((k % 64) == 0) {
5062: if ((rows % 8) == 0) {
5063: printf(
5064: " |0000 |8000 |10000 |18000 |20000 |28000 |30000 |38000\n");
5065: }
5066: printf("%08lx: ", va);
5067: }
5068:
5069: ncptes[k & 63] = 0;
5070: pte = ptep[k];
5071: if (pte == 0) {
5072: ch = '.';
5073: } else {
5074: occ--;
5075: switch (pte & 0x0c) {
5076: case 0x00:
5077: ch = 'D'; /* No cache No buff */
5078: break;
5079: case 0x04:
5080: ch = 'B'; /* No cache buff */
5081: break;
5082: case 0x08:
1.141 scw 5083: if (pte & 0x40)
5084: ch = 'm';
5085: else
5086: ch = 'C'; /* Cache No buff */
1.134 thorpej 5087: break;
5088: case 0x0c:
5089: ch = 'F'; /* Cache Buff */
5090: break;
5091: }
5092:
5093: if ((pte & L2_S_PROT_U) == L2_S_PROT_U)
5094: ch += 0x20;
5095:
5096: if ((pte & 0xc) == 0)
5097: ncptes[k & 63] = pte;
5098: }
5099:
5100: if ((k % 64) == 63) {
5101: rows++;
5102: printf("%c\n", ch);
5103: pmap_dump_ncpg(pm);
5104: if (occ == 0)
5105: break;
5106: } else
5107: printf("%c", ch);
5108: }
5109: }
5110: }
5111: }
5112:
5113: static void
5114: pmap_dump_ncpg(pmap_t pm)
5115: {
5116: struct vm_page *pg;
5117: struct pv_entry *pv;
5118: int i;
5119:
5120: for (i = 0; i < 63; i++) {
5121: if (ncptes[i] == 0)
5122: continue;
5123:
5124: pg = PHYS_TO_VM_PAGE(l2pte_pa(ncptes[i]));
5125: if (pg == NULL)
5126: continue;
5127:
5128: printf(" pa 0x%08lx: krw %d kro %d urw %d uro %d\n",
1.155 yamt 5129: VM_PAGE_TO_PHYS(pg),
1.134 thorpej 5130: pg->mdpage.krw_mappings, pg->mdpage.kro_mappings,
5131: pg->mdpage.urw_mappings, pg->mdpage.uro_mappings);
5132:
5133: for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) {
5134: printf(" %c va 0x%08lx, flags 0x%x\n",
5135: (pm == pv->pv_pmap) ? '*' : ' ',
5136: pv->pv_va, pv->pv_flags);
5137: }
5138: }
5139: }
5140: #endif
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to pmap.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / arm / arm32