Annotation of src/sys/arch/aarch64/aarch64/pmap.c, Revision 1.85
1.85 ! skrll 1: /* $NetBSD: pmap.c,v 1.84 2020/07/16 11:36:35 skrll Exp $ */
1.1 matt 2:
1.2 ryo 3: /*
4: * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
1.1 matt 5: * All rights reserved.
6: *
7: * Redistribution and use in source and binary forms, with or without
8: * modification, are permitted provided that the following conditions
9: * are met:
10: * 1. Redistributions of source code must retain the above copyright
11: * notice, this list of conditions and the following disclaimer.
12: * 2. Redistributions in binary form must reproduce the above copyright
13: * notice, this list of conditions and the following disclaimer in the
14: * documentation and/or other materials provided with the distribution.
15: *
1.2 ryo 16: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18: * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19: * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
20: * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21: * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22: * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
24: * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
25: * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
1.1 matt 26: * POSSIBILITY OF SUCH DAMAGE.
27: */
28:
29: #include <sys/cdefs.h>
1.85 ! skrll 30: __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.84 2020/07/16 11:36:35 skrll Exp $");
1.1 matt 31:
1.2 ryo 32: #include "opt_arm_debug.h"
33: #include "opt_ddb.h"
1.26 ryo 34: #include "opt_multiprocessor.h"
35: #include "opt_pmap.h"
1.2 ryo 36: #include "opt_uvmhist.h"
1.1 matt 37:
38: #include <sys/param.h>
39: #include <sys/types.h>
40: #include <sys/kmem.h>
41: #include <sys/vmem.h>
1.2 ryo 42: #include <sys/atomic.h>
1.33 maxv 43: #include <sys/asan.h>
1.1 matt 44:
45: #include <uvm/uvm.h>
1.63 ryo 46: #include <uvm/pmap/pmap_pvt.h>
1.1 matt 47:
1.2 ryo 48: #include <aarch64/pmap.h>
49: #include <aarch64/pte.h>
50: #include <aarch64/armreg.h>
51: #include <aarch64/cpufunc.h>
1.66 ryo 52: #include <aarch64/locore.h>
1.21 ryo 53: #include <aarch64/machdep.h>
1.37 ryo 54: #ifdef DDB
55: #include <aarch64/db_machdep.h>
56: #include <ddb/db_access.h>
57: #endif
1.2 ryo 58:
59: //#define PMAP_PV_DEBUG
60:
1.16 skrll 61: #ifdef VERBOSE_INIT_ARM
62: #define VPRINTF(...) printf(__VA_ARGS__)
63: #else
1.30 skrll 64: #define VPRINTF(...) __nothing
1.16 skrll 65: #endif
1.2 ryo 66:
67: UVMHIST_DEFINE(pmaphist);
68: #ifdef UVMHIST
69:
70: #ifndef UVMHIST_PMAPHIST_SIZE
71: #define UVMHIST_PMAPHIST_SIZE (1024 * 4)
72: #endif
73:
74: struct kern_history_ent pmaphistbuf[UVMHIST_PMAPHIST_SIZE];
75:
76: static void
77: pmap_hist_init(void)
78: {
79: static bool inited = false;
80: if (inited == false) {
81: UVMHIST_INIT_STATIC(pmaphist, pmaphistbuf);
82: inited = true;
83: }
84: }
85: #define PMAP_HIST_INIT() pmap_hist_init()
86:
87: #else /* UVMHIST */
88:
89: #define PMAP_HIST_INIT() ((void)0)
90:
91: #endif /* UVMHIST */
92:
93:
94: #ifdef PMAPCOUNTERS
95: #define PMAP_COUNT(name) (pmap_evcnt_##name.ev_count++ + 0)
96: #define PMAP_COUNTER(name, desc) \
97: struct evcnt pmap_evcnt_##name = \
98: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", desc); \
99: EVCNT_ATTACH_STATIC(pmap_evcnt_##name)
100:
101: PMAP_COUNTER(pdp_alloc_boot, "page table page allocate (uvm_pageboot_alloc)");
102: PMAP_COUNTER(pdp_alloc, "page table page allocate (uvm_pagealloc)");
103: PMAP_COUNTER(pdp_free, "page table page free (uvm_pagefree)");
104:
1.78 ad 105: PMAP_COUNTER(pv_enter, "pv_entry fill");
106: PMAP_COUNTER(pv_remove_dyn, "pv_entry free and unlink dynamic");
107: PMAP_COUNTER(pv_remove_emb, "pv_entry clear embedded");
1.5 ryo 108: PMAP_COUNTER(pv_remove_nopv, "no pv_entry found when removing pv");
1.2 ryo 109:
110: PMAP_COUNTER(activate, "pmap_activate call");
111: PMAP_COUNTER(deactivate, "pmap_deactivate call");
112: PMAP_COUNTER(create, "pmap_create call");
113: PMAP_COUNTER(destroy, "pmap_destroy call");
114:
115: PMAP_COUNTER(page_protect, "pmap_page_protect call");
116: PMAP_COUNTER(protect, "pmap_protect call");
117: PMAP_COUNTER(protect_remove_fallback, "pmap_protect with no-read");
118: PMAP_COUNTER(protect_none, "pmap_protect non-exists pages");
119: PMAP_COUNTER(protect_managed, "pmap_protect managed pages");
120: PMAP_COUNTER(protect_unmanaged, "pmap_protect unmanaged pages");
1.63 ryo 121: PMAP_COUNTER(protect_pvmanaged, "pmap_protect pv-tracked unmanaged pages");
1.2 ryo 122:
123: PMAP_COUNTER(clear_modify, "pmap_clear_modify call");
124: PMAP_COUNTER(clear_modify_pages, "pmap_clear_modify pages");
125: PMAP_COUNTER(clear_reference, "pmap_clear_reference call");
126: PMAP_COUNTER(clear_reference_pages, "pmap_clear_reference pages");
127:
128: PMAP_COUNTER(fixup_referenced, "page reference emulations");
129: PMAP_COUNTER(fixup_modified, "page modification emulations");
130:
131: PMAP_COUNTER(kern_mappings_bad, "kernel pages mapped (bad color)");
132: PMAP_COUNTER(kern_mappings_bad_wired, "kernel pages mapped (wired bad color)");
133: PMAP_COUNTER(user_mappings_bad, "user pages mapped (bad color, not wired)");
1.80 rin 134: PMAP_COUNTER(user_mappings_bad_wired, "user pages mapped (bad color, wired)");
1.2 ryo 135: PMAP_COUNTER(kern_mappings, "kernel pages mapped");
136: PMAP_COUNTER(user_mappings, "user pages mapped");
137: PMAP_COUNTER(user_mappings_changed, "user mapping changed");
138: PMAP_COUNTER(kern_mappings_changed, "kernel mapping changed");
139: PMAP_COUNTER(uncached_mappings, "uncached pages mapped");
140: PMAP_COUNTER(unmanaged_mappings, "unmanaged pages mapped");
1.63 ryo 141: PMAP_COUNTER(pvmanaged_mappings, "pv-tracked unmanaged pages mapped");
1.2 ryo 142: PMAP_COUNTER(managed_mappings, "managed pages mapped");
143: PMAP_COUNTER(mappings, "pages mapped (including remapped)");
144: PMAP_COUNTER(remappings, "pages remapped");
145:
146: PMAP_COUNTER(pv_entry_cannotalloc, "pv_entry allocation failure");
147:
148: PMAP_COUNTER(unwire, "pmap_unwire call");
1.3 ryo 149: PMAP_COUNTER(unwire_failure, "pmap_unwire failure");
1.2 ryo 150:
151: #else /* PMAPCOUNTERS */
152: #define PMAP_COUNT(name) __nothing
153: #endif /* PMAPCOUNTERS */
154:
1.19 ryo 155: /*
156: * invalidate TLB entry for ASID and VA.
157: * `ll' invalidates only the Last Level (usually L3) of TLB entry
158: */
159: #define AARCH64_TLBI_BY_ASID_VA(asid, va, ll) \
160: do { \
161: if ((ll)) { \
162: if ((asid) == 0) \
163: aarch64_tlbi_by_va_ll((va)); \
164: else \
165: aarch64_tlbi_by_asid_va_ll((asid), (va)); \
166: } else { \
167: if ((asid) == 0) \
168: aarch64_tlbi_by_va((va)); \
169: else \
170: aarch64_tlbi_by_asid_va((asid), (va)); \
171: } \
172: } while (0/*CONSTCOND*/)
173:
174: /*
1.76 ryo 175: * require access permission in pte to invalidate instruction cache.
176: * change the pte to accessible temporarly before cpu_icache_sync_range().
1.19 ryo 177: * this macro modifies PTE (*ptep). need to update PTE after this.
178: */
179: #define PTE_ICACHE_SYNC_PAGE(pte, ptep, pm, va, ll) \
180: do { \
1.76 ryo 181: atomic_swap_64((ptep), (pte) | LX_BLKPAG_AF); \
1.19 ryo 182: AARCH64_TLBI_BY_ASID_VA((pm)->pm_asid, (va), (ll)); \
183: cpu_icache_sync_range((va), PAGE_SIZE); \
184: } while (0/*CONSTCOND*/)
185:
1.62 ryo 186: #define VM_PAGE_TO_PP(pg) (&(pg)->mdpage.mdpg_pp)
187:
1.35 ryo 188: #define L3INDEXMASK (L3_SIZE * Ln_ENTRIES - 1)
1.40 ryo 189: #define PDPSWEEP_TRIGGER 512
1.2 ryo 190:
1.27 ryo 191: static pt_entry_t *_pmap_pte_lookup_l3(struct pmap *, vaddr_t);
192: static pt_entry_t *_pmap_pte_lookup_bs(struct pmap *, vaddr_t, vsize_t *);
1.18 ryo 193: static pt_entry_t _pmap_pte_adjust_prot(pt_entry_t, vm_prot_t, vm_prot_t, bool);
1.2 ryo 194: static pt_entry_t _pmap_pte_adjust_cacheflags(pt_entry_t, u_int);
1.36 ryo 195: static void _pmap_remove(struct pmap *, vaddr_t, vaddr_t, bool,
196: struct pv_entry **);
1.2 ryo 197: static int _pmap_enter(struct pmap *, vaddr_t, paddr_t, vm_prot_t, u_int, bool);
198:
1.78 ad 199: static struct pmap kernel_pmap __cacheline_aligned;
1.1 matt 200:
201: struct pmap * const kernel_pmap_ptr = &kernel_pmap;
1.2 ryo 202: static vaddr_t pmap_maxkvaddr;
203:
204: vaddr_t virtual_avail, virtual_end;
205: vaddr_t virtual_devmap_addr;
1.45 ryo 206: bool pmap_devmap_bootstrap_done = false;
1.2 ryo 207:
208: static struct pool_cache _pmap_cache;
209: static struct pool_cache _pmap_pv_pool;
210:
1.70 maxv 211: /* Set to LX_BLKPAG_GP if supported. */
212: uint64_t pmap_attr_gp = 0;
1.2 ryo 213:
214: static inline void
1.62 ryo 215: pmap_pv_lock(struct pmap_page *pp)
1.2 ryo 216: {
217:
1.78 ad 218: mutex_spin_enter(&pp->pp_pvlock);
1.2 ryo 219: }
220:
221: static inline void
1.62 ryo 222: pmap_pv_unlock(struct pmap_page *pp)
1.2 ryo 223: {
224:
1.78 ad 225: mutex_spin_exit(&pp->pp_pvlock);
1.2 ryo 226: }
227:
1.11 ryo 228:
229: static inline void
230: pm_lock(struct pmap *pm)
231: {
1.78 ad 232: mutex_spin_enter(&pm->pm_lock);
1.11 ryo 233: }
234:
235: static inline void
236: pm_unlock(struct pmap *pm)
237: {
1.78 ad 238: mutex_spin_exit(&pm->pm_lock);
239: }
240:
241: static bool
242: pm_reverse_lock(struct pmap *pm, struct pmap_page *pp)
243: {
244:
245: KASSERT(mutex_owned(&pp->pp_pvlock));
246:
247: if (__predict_true(mutex_tryenter(&pm->pm_lock)))
248: return true;
249:
250: if (pm != pmap_kernel())
251: pmap_reference(pm);
252: mutex_spin_exit(&pp->pp_pvlock);
253: mutex_spin_enter(&pm->pm_lock);
254: /* nothing, just wait for lock */
255: mutex_spin_exit(&pm->pm_lock);
256: if (pm != pmap_kernel())
257: pmap_destroy(pm);
258: mutex_spin_enter(&pp->pp_pvlock);
259: return false;
1.11 ryo 260: }
1.2 ryo 261:
1.62 ryo 262: static inline struct pmap_page *
263: phys_to_pp(paddr_t pa)
264: {
265: struct vm_page *pg;
266:
267: pg = PHYS_TO_VM_PAGE(pa);
268: if (pg != NULL)
269: return VM_PAGE_TO_PP(pg);
270:
1.63 ryo 271: #ifdef __HAVE_PMAP_PV_TRACK
1.69 ryo 272: return pmap_pv_tracked(pa);
1.63 ryo 273: #else
1.62 ryo 274: return NULL;
1.63 ryo 275: #endif /* __HAVE_PMAP_PV_TRACK */
1.62 ryo 276: }
277:
1.28 ryo 278: #define IN_RANGE(va,sta,end) (((sta) <= (va)) && ((va) < (end)))
279:
1.2 ryo 280: #define IN_KSEG_ADDR(va) \
1.28 ryo 281: IN_RANGE((va), AARCH64_KSEG_START, AARCH64_KSEG_END)
282:
1.66 ryo 283: #ifdef DIAGNOSTIC
284: #define KASSERT_PM_ADDR(pm,va) \
1.28 ryo 285: do { \
1.66 ryo 286: int space = aarch64_addressspace(va); \
1.28 ryo 287: if ((pm) == pmap_kernel()) { \
1.66 ryo 288: KASSERTMSG(space == AARCH64_ADDRSPACE_UPPER, \
289: "%s: kernel pm %p: va=%016lx" \
290: " is out of upper address space\n", \
291: __func__, (pm), (va)); \
1.28 ryo 292: KASSERTMSG(IN_RANGE((va), VM_MIN_KERNEL_ADDRESS, \
293: VM_MAX_KERNEL_ADDRESS), \
294: "%s: kernel pm %p: va=%016lx" \
295: " is not kernel address\n", \
296: __func__, (pm), (va)); \
297: } else { \
1.66 ryo 298: KASSERTMSG(space == AARCH64_ADDRSPACE_LOWER, \
299: "%s: user pm %p: va=%016lx" \
300: " is out of lower address space\n", \
301: __func__, (pm), (va)); \
1.28 ryo 302: KASSERTMSG(IN_RANGE((va), \
303: VM_MIN_ADDRESS, VM_MAX_ADDRESS), \
304: "%s: user pm %p: va=%016lx" \
305: " is not user address\n", \
306: __func__, (pm), (va)); \
307: } \
308: } while (0 /* CONSTCOND */)
1.66 ryo 309: #else /* DIAGNOSTIC */
310: #define KASSERT_PM_ADDR(pm,va)
311: #endif /* DIAGNOSTIC */
1.2 ryo 312:
313:
314: static const struct pmap_devmap *pmap_devmap_table;
315:
316: static vsize_t
1.26 ryo 317: pmap_map_chunk(vaddr_t va, paddr_t pa, vsize_t size,
1.2 ryo 318: vm_prot_t prot, u_int flags)
319: {
320: pt_entry_t attr;
1.60 skrll 321: vsize_t resid = round_page(size);
322:
1.65 ryo 323: attr = _pmap_pte_adjust_prot(0, prot, VM_PROT_ALL, false);
324: attr = _pmap_pte_adjust_cacheflags(attr, flags);
1.84 skrll 325: pmapboot_enter_range(va, pa, resid, attr, printf);
1.65 ryo 326: aarch64_tlbi_all();
1.2 ryo 327:
1.65 ryo 328: return resid;
1.2 ryo 329: }
1.1 matt 330:
331: void
1.2 ryo 332: pmap_devmap_register(const struct pmap_devmap *table)
1.1 matt 333: {
1.2 ryo 334: pmap_devmap_table = table;
1.1 matt 335: }
336:
337: void
1.31 skrll 338: pmap_devmap_bootstrap(vaddr_t l0pt, const struct pmap_devmap *table)
1.2 ryo 339: {
340: vaddr_t va;
341: int i;
342:
343: pmap_devmap_register(table);
344:
1.16 skrll 345: VPRINTF("%s:\n", __func__);
1.2 ryo 346: for (i = 0; table[i].pd_size != 0; i++) {
1.16 skrll 347: VPRINTF(" devmap: pa %08lx-%08lx = va %016lx\n",
1.2 ryo 348: table[i].pd_pa,
349: table[i].pd_pa + table[i].pd_size - 1,
350: table[i].pd_va);
351: va = table[i].pd_va;
352:
1.26 ryo 353: KASSERT((VM_KERNEL_IO_ADDRESS <= va) &&
354: (va < (VM_KERNEL_IO_ADDRESS + VM_KERNEL_IO_SIZE)));
355:
1.2 ryo 356: /* update and check virtual_devmap_addr */
1.59 skrll 357: if (virtual_devmap_addr == 0 || virtual_devmap_addr > va) {
1.2 ryo 358: virtual_devmap_addr = va;
359: }
360:
1.26 ryo 361: pmap_map_chunk(
1.2 ryo 362: table[i].pd_va,
363: table[i].pd_pa,
364: table[i].pd_size,
365: table[i].pd_prot,
366: table[i].pd_flags);
367: }
1.45 ryo 368:
369: pmap_devmap_bootstrap_done = true;
1.2 ryo 370: }
371:
372: const struct pmap_devmap *
373: pmap_devmap_find_va(vaddr_t va, vsize_t size)
374: {
375: paddr_t endva;
376: int i;
377:
378: if (pmap_devmap_table == NULL)
379: return NULL;
380:
381: endva = va + size;
382: for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) {
383: if ((va >= pmap_devmap_table[i].pd_va) &&
384: (endva <= pmap_devmap_table[i].pd_va +
385: pmap_devmap_table[i].pd_size))
386: return &pmap_devmap_table[i];
387: }
388: return NULL;
389: }
390:
391: const struct pmap_devmap *
392: pmap_devmap_find_pa(paddr_t pa, psize_t size)
393: {
394: paddr_t endpa;
395: int i;
396:
397: if (pmap_devmap_table == NULL)
398: return NULL;
399:
400: endpa = pa + size;
401: for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) {
402: if (pa >= pmap_devmap_table[i].pd_pa &&
403: (endpa <= pmap_devmap_table[i].pd_pa +
404: pmap_devmap_table[i].pd_size))
405: return (&pmap_devmap_table[i]);
406: }
407: return NULL;
408: }
409:
410: vaddr_t
411: pmap_devmap_phystov(paddr_t pa)
1.1 matt 412: {
1.2 ryo 413: const struct pmap_devmap *table;
414: paddr_t offset;
415:
416: table = pmap_devmap_find_pa(pa, 0);
417: if (table == NULL)
418: return 0;
419:
420: offset = pa - table->pd_pa;
421: return table->pd_va + offset;
1.1 matt 422: }
423:
424: vaddr_t
1.2 ryo 425: pmap_devmap_vtophys(paddr_t va)
426: {
427: const struct pmap_devmap *table;
428: vaddr_t offset;
429:
430: table = pmap_devmap_find_va(va, 0);
431: if (table == NULL)
432: return 0;
433:
434: offset = va - table->pd_va;
435: return table->pd_pa + offset;
436: }
437:
438: void
439: pmap_bootstrap(vaddr_t vstart, vaddr_t vend)
440: {
441: struct pmap *kpm;
442: pd_entry_t *l0;
443: paddr_t l0pa;
444:
445: PMAP_HIST_INIT(); /* init once */
446:
447: UVMHIST_FUNC(__func__);
448: UVMHIST_CALLED(pmaphist);
449:
450: uvmexp.ncolors = aarch64_cache_vindexsize / PAGE_SIZE;
451:
452: /* devmap already uses last of va? */
1.59 skrll 453: if (virtual_devmap_addr != 0 && virtual_devmap_addr < vend)
1.2 ryo 454: vend = virtual_devmap_addr;
455:
456: virtual_avail = vstart;
457: virtual_end = vend;
458: pmap_maxkvaddr = vstart;
459:
460: aarch64_tlbi_all();
461:
462: l0pa = reg_ttbr1_el1_read();
1.9 christos 463: l0 = (void *)AARCH64_PA_TO_KVA(l0pa);
1.2 ryo 464:
465: memset(&kernel_pmap, 0, sizeof(kernel_pmap));
466: kpm = pmap_kernel();
467: kpm->pm_asid = 0;
468: kpm->pm_refcnt = 1;
1.40 ryo 469: kpm->pm_idlepdp = 0;
1.2 ryo 470: kpm->pm_l0table = l0;
471: kpm->pm_l0table_pa = l0pa;
472: kpm->pm_activated = true;
1.64 ryo 473: LIST_INIT(&kpm->pm_vmlist);
1.2 ryo 474: mutex_init(&kpm->pm_lock, MUTEX_DEFAULT, IPL_VM);
1.36 ryo 475:
476: CTASSERT(sizeof(kpm->pm_stats.wired_count) == sizeof(long));
477: CTASSERT(sizeof(kpm->pm_stats.resident_count) == sizeof(long));
1.78 ad 478: #define PMSTAT_INC_WIRED_COUNT(pm) do { \
479: KASSERT(mutex_owned(&(pm)->pm_lock)); \
480: (pm)->pm_stats.wired_count++; \
481: } while (/* CONSTCOND */ 0);
482: #define PMSTAT_DEC_WIRED_COUNT(pm) do{ \
483: KASSERT(mutex_owned(&(pm)->pm_lock)); \
484: (pm)->pm_stats.wired_count--; \
485: } while (/* CONSTCOND */ 0);
486: #define PMSTAT_INC_RESIDENT_COUNT(pm) do { \
487: KASSERT(mutex_owned(&(pm)->pm_lock)); \
488: (pm)->pm_stats.resident_count++; \
489: } while (/* CONSTCOND */ 0);
490: #define PMSTAT_DEC_RESIDENT_COUNT(pm) do { \
491: KASSERT(mutex_owned(&(pm)->pm_lock)); \
492: (pm)->pm_stats.resident_count--; \
493: } while (/* CONSTCOND */ 0);
1.2 ryo 494: }
495:
496: inline static int
497: _pmap_color(vaddr_t addr) /* or paddr_t */
498: {
499: return (addr >> PGSHIFT) & (uvmexp.ncolors - 1);
500: }
501:
502: static int
503: _pmap_pmap_ctor(void *arg, void *v, int flags)
1.1 matt 504: {
1.2 ryo 505: memset(v, 0, sizeof(struct pmap));
1.1 matt 506: return 0;
507: }
508:
1.2 ryo 509: static int
510: _pmap_pv_ctor(void *arg, void *v, int flags)
1.1 matt 511: {
1.2 ryo 512: memset(v, 0, sizeof(struct pv_entry));
513: return 0;
1.1 matt 514: }
515:
516: void
1.2 ryo 517: pmap_init(void)
1.1 matt 518: {
1.2 ryo 519:
520: pool_cache_bootstrap(&_pmap_cache, sizeof(struct pmap),
1.78 ad 521: coherency_unit, 0, 0, "pmappl", NULL, IPL_NONE, _pmap_pmap_ctor,
522: NULL, NULL);
523:
1.2 ryo 524: pool_cache_bootstrap(&_pmap_pv_pool, sizeof(struct pv_entry),
1.78 ad 525: 32, 0, PR_LARGECACHE, "pvpl", NULL, IPL_NONE, _pmap_pv_ctor,
526: NULL, NULL);
1.2 ryo 527: }
528:
529: void
530: pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp)
531: {
532: *vstartp = virtual_avail;
533: *vendp = virtual_end;
534: }
535:
536: vaddr_t
537: pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp)
538: {
539: int npage;
540: paddr_t pa;
541: vaddr_t va;
542: psize_t bank_npage;
543: uvm_physseg_t bank;
544:
545: UVMHIST_FUNC(__func__);
546: UVMHIST_CALLED(pmaphist);
547:
548: UVMHIST_LOG(pmaphist, "size=%llu, *vstartp=%llx, *vendp=%llx",
549: size, *vstartp, *vendp, 0);
550:
551: size = round_page(size);
552: npage = atop(size);
553:
554: for (bank = uvm_physseg_get_first(); uvm_physseg_valid_p(bank);
555: bank = uvm_physseg_get_next(bank)) {
556:
557: bank_npage = uvm_physseg_get_avail_end(bank) -
558: uvm_physseg_get_avail_start(bank);
559: if (npage <= bank_npage)
560: break;
561: }
562:
1.23 maxv 563: if (!uvm_physseg_valid_p(bank)) {
564: panic("%s: no memory", __func__);
565: }
1.2 ryo 566:
567: /* Steal pages */
568: pa = ptoa(uvm_physseg_get_avail_start(bank));
569: va = AARCH64_PA_TO_KVA(pa);
570: uvm_physseg_unplug(atop(pa), npage);
571:
572: for (; npage > 0; npage--, pa += PAGE_SIZE)
573: pmap_zero_page(pa);
574:
575: return va;
1.1 matt 576: }
577:
578: void
579: pmap_reference(struct pmap *pm)
580: {
1.2 ryo 581: atomic_inc_uint(&pm->pm_refcnt);
1.1 matt 582: }
583:
1.36 ryo 584: paddr_t
1.40 ryo 585: pmap_alloc_pdp(struct pmap *pm, struct vm_page **pgp, int flags, bool waitok)
1.1 matt 586: {
1.2 ryo 587: paddr_t pa;
1.36 ryo 588: struct vm_page *pg;
1.2 ryo 589:
590: UVMHIST_FUNC(__func__);
591: UVMHIST_CALLED(pmaphist);
592:
593: if (uvm.page_init_done) {
1.40 ryo 594: int aflags = ((flags & PMAP_CANFAIL) ? 0 : UVM_PGA_USERESERVE) |
595: UVM_PGA_ZERO;
1.36 ryo 596: retry:
1.40 ryo 597: pg = uvm_pagealloc(NULL, 0, NULL, aflags);
1.36 ryo 598: if (pg == NULL) {
599: if (waitok) {
600: uvm_wait("pmap_alloc_pdp");
601: goto retry;
602: }
603: return POOL_PADDR_INVALID;
604: }
605:
1.78 ad 606: LIST_INSERT_HEAD(&pm->pm_vmlist, pg, pageq.list);
1.36 ryo 607: pg->flags &= ~PG_BUSY; /* never busy */
608: pg->wire_count = 1; /* max = 1 + Ln_ENTRIES = 513 */
1.2 ryo 609: pa = VM_PAGE_TO_PHYS(pg);
1.36 ryo 610: PMAP_COUNT(pdp_alloc);
1.78 ad 611: PMAP_PAGE_INIT(VM_PAGE_TO_PP(pg));
1.2 ryo 612: } else {
613: /* uvm_pageboot_alloc() returns AARCH64 KSEG address */
1.36 ryo 614: pg = NULL;
1.2 ryo 615: pa = AARCH64_KVA_TO_PA(
616: uvm_pageboot_alloc(Ln_TABLE_SIZE));
617: PMAP_COUNT(pdp_alloc_boot);
618: }
1.36 ryo 619: if (pgp != NULL)
620: *pgp = pg;
621:
622: UVMHIST_LOG(pmaphist, "pa=%llx, pg=%llx",
623: pa, pg, 0, 0);
624:
625: return pa;
626: }
1.2 ryo 627:
1.36 ryo 628: static void
629: pmap_free_pdp(struct pmap *pm, struct vm_page *pg)
630: {
1.73 skrll 631:
1.78 ad 632: KASSERT(pm != pmap_kernel());
633: KASSERT(VM_PAGE_TO_PP(pg)->pp_pv.pv_pmap == NULL);
634: KASSERT(VM_PAGE_TO_PP(pg)->pp_pv.pv_next == NULL);
1.2 ryo 635:
1.78 ad 636: LIST_REMOVE(pg, pageq.list);
637: pg->wire_count = 0;
1.36 ryo 638: uvm_pagefree(pg);
639: PMAP_COUNT(pdp_free);
1.1 matt 640: }
641:
1.40 ryo 642: /* free empty page table pages */
643: static int
644: _pmap_sweep_pdp(struct pmap *pm)
645: {
646: struct vm_page *pg, *tmp;
1.41 mrg 647: pd_entry_t *ptep_in_parent, opte __diagused;
1.40 ryo 648: paddr_t pa, pdppa;
649: int nsweep;
1.41 mrg 650: uint16_t wirecount __diagused;
1.40 ryo 651:
1.78 ad 652: KASSERT(mutex_owned(&pm->pm_lock) || pm->pm_refcnt == 0);
653:
1.40 ryo 654: nsweep = 0;
1.78 ad 655: LIST_FOREACH_SAFE(pg, &pm->pm_vmlist, pageq.list, tmp) {
1.40 ryo 656: if (pg->wire_count != 1)
657: continue;
658:
659: pa = VM_PAGE_TO_PHYS(pg);
660: if (pa == pm->pm_l0table_pa)
661: continue;
662:
663: ptep_in_parent = VM_PAGE_TO_MD(pg)->mdpg_ptep_parent;
664: if (ptep_in_parent == NULL) {
665: /* no parent */
666: pmap_free_pdp(pm, pg);
667: nsweep++;
668: continue;
669: }
670:
671: /* unlink from parent */
672: opte = atomic_swap_64(ptep_in_parent, 0);
673: KASSERT(lxpde_valid(opte));
1.78 ad 674: wirecount = --pg->wire_count; /* 1 -> 0 */
1.40 ryo 675: KASSERT(wirecount == 0);
676: pmap_free_pdp(pm, pg);
677: nsweep++;
678:
679: /* L3->L2->L1. no need for L0 */
680: pdppa = AARCH64_KVA_TO_PA(trunc_page((vaddr_t)ptep_in_parent));
681: if (pdppa == pm->pm_l0table_pa)
682: continue;
683:
684: pg = PHYS_TO_VM_PAGE(pdppa);
685: KASSERT(pg != NULL);
686: KASSERTMSG(pg->wire_count >= 1,
687: "wire_count=%d", pg->wire_count);
688: /* decrement wire_count of parent */
1.78 ad 689: wirecount = --pg->wire_count;
1.40 ryo 690: KASSERTMSG(pg->wire_count <= (Ln_ENTRIES + 1),
691: "pm=%p[%d], pg=%p, wire_count=%d",
692: pm, pm->pm_asid, pg, pg->wire_count);
693: }
1.78 ad 694: pm->pm_idlepdp = 0;
1.40 ryo 695:
696: return nsweep;
697: }
698:
1.2 ryo 699: static void
700: _pmap_free_pdp_all(struct pmap *pm)
1.1 matt 701: {
1.78 ad 702: struct vm_page *pg;
1.2 ryo 703:
1.78 ad 704: while ((pg = LIST_FIRST(&pm->pm_vmlist)) != NULL) {
1.36 ryo 705: pmap_free_pdp(pm, pg);
1.2 ryo 706: }
1.1 matt 707: }
708:
709: vaddr_t
710: pmap_growkernel(vaddr_t maxkvaddr)
711: {
1.2 ryo 712: UVMHIST_FUNC(__func__);
713: UVMHIST_CALLED(pmaphist);
714:
715: UVMHIST_LOG(pmaphist, "maxkvaddr=%llx, pmap_maxkvaddr=%llx",
716: maxkvaddr, pmap_maxkvaddr, 0, 0);
717:
1.54 ryo 718: kasan_shadow_map((void *)pmap_maxkvaddr,
719: (size_t)(maxkvaddr - pmap_maxkvaddr));
1.52 skrll 720:
1.2 ryo 721: pmap_maxkvaddr = maxkvaddr;
722:
1.1 matt 723: return maxkvaddr;
724: }
725:
1.2 ryo 726: bool
1.72 jmcneill 727: pmap_extract(struct pmap *pm, vaddr_t va, paddr_t *pap)
1.2 ryo 728: {
729:
1.72 jmcneill 730: return pmap_extract_coherency(pm, va, pap, NULL);
1.2 ryo 731: }
732:
733: bool
1.72 jmcneill 734: pmap_extract_coherency(struct pmap *pm, vaddr_t va, paddr_t *pap,
735: bool *coherencyp)
1.2 ryo 736: {
1.36 ryo 737: pt_entry_t *ptep, pte;
1.2 ryo 738: paddr_t pa;
1.27 ryo 739: vsize_t blocksize = 0;
1.66 ryo 740: int space;
1.72 jmcneill 741: bool coherency;
1.28 ryo 742: extern char __kernel_text[];
743: extern char _end[];
1.2 ryo 744:
1.72 jmcneill 745: coherency = false;
746:
1.66 ryo 747: space = aarch64_addressspace(va);
748: if (pm == pmap_kernel()) {
749: if (space != AARCH64_ADDRSPACE_UPPER)
750: return false;
751:
752: if (IN_RANGE(va, (vaddr_t)__kernel_text, (vaddr_t)_end)) {
753: /* kernel text/data/bss are definitely linear mapped */
754: pa = KERN_VTOPHYS(va);
755: goto mapped;
756: } else if (IN_KSEG_ADDR(va)) {
757: /*
758: * also KSEG is linear mapped, but areas that have no
759: * physical memory haven't been mapped.
760: * fast lookup by using the S1E1R/PAR_EL1 registers.
761: */
762: register_t s = daif_disable(DAIF_I|DAIF_F);
763: reg_s1e1r_write(va);
764: __asm __volatile ("isb");
765: uint64_t par = reg_par_el1_read();
1.67 ryo 766: reg_daif_write(s);
1.66 ryo 767:
768: if (par & PAR_F)
769: return false;
770: pa = (__SHIFTOUT(par, PAR_PA) << PAR_PA_SHIFT) +
771: (va & __BITS(PAR_PA_SHIFT - 1, 0));
772: goto mapped;
773: }
1.28 ryo 774: } else {
1.66 ryo 775: if (space != AARCH64_ADDRSPACE_LOWER)
1.35 ryo 776: return false;
1.28 ryo 777: }
1.2 ryo 778:
1.66 ryo 779: /*
780: * other areas, it isn't able to examined using the PAR_EL1 register,
781: * because the page may be in an access fault state due to
782: * reference bit emulation.
783: */
784: ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
785: if (ptep == NULL)
786: return false;
787: pte = *ptep;
788: if (!lxpde_valid(pte))
789: return false;
790: pa = lxpde_pa(pte) + (va & (blocksize - 1));
791:
1.72 jmcneill 792: switch (pte & LX_BLKPAG_ATTR_MASK) {
793: case LX_BLKPAG_ATTR_NORMAL_NC:
794: case LX_BLKPAG_ATTR_DEVICE_MEM:
795: case LX_BLKPAG_ATTR_DEVICE_MEM_SO:
796: coherency = true;
797: break;
798: }
799:
1.66 ryo 800: mapped:
1.27 ryo 801: if (pap != NULL)
1.2 ryo 802: *pap = pa;
1.72 jmcneill 803: if (coherencyp != NULL)
804: *coherencyp = coherency;
1.27 ryo 805: return true;
1.2 ryo 806: }
807:
808: paddr_t
809: vtophys(vaddr_t va)
810: {
811: struct pmap *pm;
812: paddr_t pa;
813:
1.66 ryo 814: /* even if TBI is disabled, AARCH64_ADDRTOP_TAG means KVA */
815: if ((uint64_t)va & AARCH64_ADDRTOP_TAG)
1.28 ryo 816: pm = pmap_kernel();
817: else
1.2 ryo 818: pm = curlwp->l_proc->p_vmspace->vm_map.pmap;
1.28 ryo 819:
820: if (pmap_extract(pm, va, &pa) == false)
1.2 ryo 821: return VTOPHYS_FAILED;
822: return pa;
823: }
824:
1.36 ryo 825: /*
826: * return pointer of the pte. regardess of whether the entry is valid or not.
827: */
1.2 ryo 828: static pt_entry_t *
1.27 ryo 829: _pmap_pte_lookup_bs(struct pmap *pm, vaddr_t va, vsize_t *bs)
1.2 ryo 830: {
1.27 ryo 831: pt_entry_t *ptep;
832: pd_entry_t *l0, *l1, *l2, *l3;
833: pd_entry_t pde;
834: vsize_t blocksize;
835: unsigned int idx;
836:
837: /*
838: * traverse L0 -> L1 -> L2 -> L3
839: */
840: blocksize = L0_SIZE;
841: l0 = pm->pm_l0table;
842: idx = l0pde_index(va);
1.35 ryo 843: ptep = &l0[idx];
844: pde = *ptep;
845: if (!l0pde_valid(pde))
1.27 ryo 846: goto done;
847:
848: blocksize = L1_SIZE;
849: l1 = (pd_entry_t *)AARCH64_PA_TO_KVA(l0pde_pa(pde));
850: idx = l1pde_index(va);
1.35 ryo 851: ptep = &l1[idx];
852: pde = *ptep;
853: if (!l1pde_valid(pde) || l1pde_is_block(pde))
1.27 ryo 854: goto done;
855:
856: blocksize = L2_SIZE;
857: l2 = (pd_entry_t *)AARCH64_PA_TO_KVA(l1pde_pa(pde));
858: idx = l2pde_index(va);
1.35 ryo 859: ptep = &l2[idx];
860: pde = *ptep;
861: if (!l2pde_valid(pde) || l2pde_is_block(pde))
1.27 ryo 862: goto done;
1.2 ryo 863:
1.27 ryo 864: blocksize = L3_SIZE;
865: l3 = (pd_entry_t *)AARCH64_PA_TO_KVA(l2pde_pa(pde));
866: idx = l3pte_index(va);
867: ptep = &l3[idx];
1.2 ryo 868:
1.27 ryo 869: done:
870: if (bs != NULL)
871: *bs = blocksize;
872: return ptep;
873: }
1.2 ryo 874:
1.27 ryo 875: static pt_entry_t *
876: _pmap_pte_lookup_l3(struct pmap *pm, vaddr_t va)
877: {
878: pt_entry_t *ptep;
879: vsize_t blocksize = 0;
880:
881: ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
882: if ((ptep != NULL) && (blocksize == L3_SIZE))
1.2 ryo 883: return ptep;
884:
885: return NULL;
886: }
887:
1.29 ryo 888: void
889: pmap_icache_sync_range(pmap_t pm, vaddr_t sva, vaddr_t eva)
890: {
1.35 ryo 891: pt_entry_t *ptep = NULL, pte;
1.29 ryo 892: vaddr_t va;
893: vsize_t blocksize = 0;
894:
1.66 ryo 895: KASSERT_PM_ADDR(pm, sva);
896:
1.29 ryo 897: pm_lock(pm);
898:
1.35 ryo 899: for (va = sva; va < eva; va = (va + blocksize) & ~(blocksize - 1)) {
900: /* va is belong to the same L3 table as before? */
901: if ((blocksize == L3_SIZE) && ((va & L3INDEXMASK) != 0)) {
902: ptep++;
903: } else {
904: ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
905: if (ptep == NULL)
906: break;
907: }
908:
909: pte = *ptep;
910: if (lxpde_valid(pte)) {
1.29 ryo 911: vaddr_t eob = (va + blocksize) & ~(blocksize - 1);
1.79 ryo 912: vsize_t len = ulmin(eva, eob) - va;
1.29 ryo 913:
914: if (l3pte_writable(pte)) {
915: cpu_icache_sync_range(va, len);
916: } else {
917: /*
918: * change to writable temporally
919: * to do cpu_icache_sync_range()
920: */
921: pt_entry_t opte = pte;
922: pte = pte & ~(LX_BLKPAG_AF|LX_BLKPAG_AP);
923: pte |= (LX_BLKPAG_AF|LX_BLKPAG_AP_RW);
924: atomic_swap_64(ptep, pte);
925: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
926: cpu_icache_sync_range(va, len);
927: atomic_swap_64(ptep, opte);
928: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
929: }
930: }
931: }
932:
933: pm_unlock(pm);
934: }
935:
1.48 maya 936: /*
937: * Routine: pmap_procwr
938: *
939: * Function:
940: * Synchronize caches corresponding to [addr, addr+len) in p.
941: *
942: */
943: void
1.82 rin 944: pmap_procwr(struct proc *p, vaddr_t sva, int len)
1.48 maya 945: {
946:
1.82 rin 947: if (__predict_true(p == curproc))
948: cpu_icache_sync_range(sva, len);
949: else {
950: struct pmap *pm = p->p_vmspace->vm_map.pmap;
951: paddr_t pa;
952: vaddr_t va, eva;
1.83 rin 953: int tlen;
1.82 rin 954:
1.83 rin 955: for (va = sva; len > 0; va = eva, len -= tlen) {
1.82 rin 956: eva = uimin(va + len, trunc_page(va + PAGE_SIZE));
1.83 rin 957: tlen = eva - va;
1.82 rin 958: if (!pmap_extract(pm, va, &pa))
959: continue;
960: va = AARCH64_PA_TO_KVA(pa);
1.83 rin 961: cpu_icache_sync_range(va, tlen);
1.82 rin 962: }
963: }
1.48 maya 964: }
965:
1.2 ryo 966: static pt_entry_t
1.18 ryo 967: _pmap_pte_adjust_prot(pt_entry_t pte, vm_prot_t prot, vm_prot_t protmask,
968: bool user)
1.1 matt 969: {
1.2 ryo 970: vm_prot_t masked;
1.18 ryo 971: pt_entry_t xn;
1.2 ryo 972:
973: masked = prot & protmask;
974: pte &= ~(LX_BLKPAG_OS_RWMASK|LX_BLKPAG_AF|LX_BLKPAG_AP);
975:
976: /* keep prot for ref/mod emulation */
977: switch (prot & (VM_PROT_READ|VM_PROT_WRITE)) {
978: case 0:
979: default:
980: break;
981: case VM_PROT_READ:
982: pte |= LX_BLKPAG_OS_READ;
983: break;
984: case VM_PROT_WRITE:
985: case VM_PROT_READ|VM_PROT_WRITE:
986: pte |= (LX_BLKPAG_OS_READ|LX_BLKPAG_OS_WRITE);
987: break;
988: }
989:
990: switch (masked & (VM_PROT_READ|VM_PROT_WRITE)) {
991: case 0:
992: default:
993: /* cannot access due to No LX_BLKPAG_AF */
994: pte |= LX_BLKPAG_AP_RO;
995: break;
996: case VM_PROT_READ:
997: /* actual permission of pte */
998: pte |= LX_BLKPAG_AF;
999: pte |= LX_BLKPAG_AP_RO;
1000: break;
1001: case VM_PROT_WRITE:
1002: case VM_PROT_READ|VM_PROT_WRITE:
1003: /* actual permission of pte */
1004: pte |= LX_BLKPAG_AF;
1005: pte |= LX_BLKPAG_AP_RW;
1006: break;
1007: }
1008:
1.18 ryo 1009: /* executable for kernel or user? first set never exec both */
1010: pte |= (LX_BLKPAG_UXN|LX_BLKPAG_PXN);
1011: /* and either to executable */
1012: xn = user ? LX_BLKPAG_UXN : LX_BLKPAG_PXN;
1013: if (prot & VM_PROT_EXECUTE)
1.35 ryo 1014: pte &= ~xn;
1.2 ryo 1015:
1016: return pte;
1017: }
1018:
1019: static pt_entry_t
1020: _pmap_pte_adjust_cacheflags(pt_entry_t pte, u_int flags)
1021: {
1022:
1023: pte &= ~LX_BLKPAG_ATTR_MASK;
1024:
1.58 jmcneill 1025: switch (flags & (PMAP_CACHE_MASK|PMAP_DEV_MASK)) {
1026: case PMAP_DEV_SO ... PMAP_DEV_SO | PMAP_CACHE_MASK:
1027: pte |= LX_BLKPAG_ATTR_DEVICE_MEM_SO; /* Device-nGnRnE */
1028: break;
1.2 ryo 1029: case PMAP_DEV ... PMAP_DEV | PMAP_CACHE_MASK:
1.57 jmcneill 1030: pte |= LX_BLKPAG_ATTR_DEVICE_MEM; /* Device-nGnRE */
1.2 ryo 1031: break;
1032: case PMAP_NOCACHE:
1033: case PMAP_NOCACHE_OVR:
1034: case PMAP_WRITE_COMBINE:
1035: pte |= LX_BLKPAG_ATTR_NORMAL_NC; /* only no-cache */
1036: break;
1037: case PMAP_WRITE_BACK:
1038: case 0:
1039: default:
1040: pte |= LX_BLKPAG_ATTR_NORMAL_WB;
1041: break;
1042: }
1043:
1044: return pte;
1045: }
1046:
1.14 ryo 1047: static struct pv_entry *
1.78 ad 1048: _pmap_remove_pv(struct pmap_page *pp, struct pmap *pm, vaddr_t va,
1049: pt_entry_t pte)
1.2 ryo 1050: {
1.78 ad 1051: struct pv_entry *pv, *ppv;
1.2 ryo 1052:
1053: UVMHIST_FUNC(__func__);
1054: UVMHIST_CALLED(pmaphist);
1055:
1.62 ryo 1056: UVMHIST_LOG(pmaphist, "pp=%p, pm=%p, va=%llx, pte=%llx",
1057: pp, pm, va, pte);
1.2 ryo 1058:
1.78 ad 1059: KASSERT(mutex_owned(&pp->pp_pvlock));
1060:
1061: for (ppv = NULL, pv = &pp->pp_pv; pv != NULL; pv = pv->pv_next) {
1062: if (pv->pv_pmap == pm && trunc_page(pv->pv_va) == va) {
1.2 ryo 1063: break;
1064: }
1.78 ad 1065: ppv = pv;
1.2 ryo 1066: }
1.78 ad 1067: if (ppv == NULL) {
1068: /* embedded in pmap_page */
1069: pv->pv_pmap = NULL;
1070: pv = NULL;
1071: PMAP_COUNT(pv_remove_emb);
1072: } else if (pv != NULL) {
1073: /* dynamically allocated */
1074: ppv->pv_next = pv->pv_next;
1075: PMAP_COUNT(pv_remove_dyn);
1076: } else {
1.5 ryo 1077: PMAP_COUNT(pv_remove_nopv);
1078: }
1.2 ryo 1079:
1.14 ryo 1080: return pv;
1.2 ryo 1081: }
1082:
1083: #if defined(PMAP_PV_DEBUG) || defined(DDB)
1084:
1085: static char *
1086: str_vmflags(uint32_t flags)
1087: {
1088: static int idx = 0;
1089: static char buf[4][32]; /* XXX */
1090: char *p;
1091:
1092: p = buf[idx];
1093: idx = (idx + 1) & 3;
1094:
1095: p[0] = (flags & VM_PROT_READ) ? 'R' : '-';
1096: p[1] = (flags & VM_PROT_WRITE) ? 'W' : '-';
1097: p[2] = (flags & VM_PROT_EXECUTE) ? 'X' : '-';
1098: if (flags & PMAP_WIRED)
1099: memcpy(&p[3], ",WIRED\0", 7);
1100: else
1101: p[3] = '\0';
1102:
1103: return p;
1104: }
1105:
1106: static void
1.38 ryo 1107: pg_dump(struct vm_page *pg, void (*pr)(const char *, ...) __printflike(1, 2))
1.2 ryo 1108: {
1109: pr("pg=%p\n", pg);
1110: pr(" pg->uanon = %p\n", pg->uanon);
1111: pr(" pg->uobject = %p\n", pg->uobject);
1112: pr(" pg->offset = %zu\n", pg->offset);
1113: pr(" pg->flags = %u\n", pg->flags);
1114: pr(" pg->loan_count = %u\n", pg->loan_count);
1115: pr(" pg->wire_count = %u\n", pg->wire_count);
1116: pr(" pg->pqflags = %u\n", pg->pqflags);
1.51 ad 1117: pr(" pg->phys_addr = %016lx\n", VM_PAGE_TO_PHYS(pg));
1.2 ryo 1118: }
1119:
1120: static void
1.62 ryo 1121: pv_dump(struct pmap_page *pp, void (*pr)(const char *, ...) __printflike(1, 2))
1.2 ryo 1122: {
1123: struct pv_entry *pv;
1.78 ad 1124: int i, flags;
1.2 ryo 1125:
1126: i = 0;
1.78 ad 1127: flags = pp->pp_pv.pv_va & (PAGE_SIZE - 1);
1.2 ryo 1128:
1.62 ryo 1129: pr("pp=%p\n", pp);
1.78 ad 1130: pr(" pp flags=%08x %s\n", flags, str_vmflags(flags));
1.2 ryo 1131:
1.78 ad 1132: for (pv = &pp->pp_pv; pv != NULL; pv = pv->pv_next) {
1133: if (pv->pv_pmap == NULL) {
1134: KASSERT(pv == &pp->pp_pv);
1135: continue;
1136: }
1.2 ryo 1137: pr(" pv[%d] pv=%p\n",
1138: i, pv);
1.35 ryo 1139: pr(" pv[%d].pv_pmap = %p (asid=%d)\n",
1.2 ryo 1140: i, pv->pv_pmap, pv->pv_pmap->pm_asid);
1.35 ryo 1141: pr(" pv[%d].pv_va = %016lx (color=%d)\n",
1.78 ad 1142: i, trunc_page(pv->pv_va), _pmap_color(pv->pv_va));
1.35 ryo 1143: pr(" pv[%d].pv_ptep = %p\n",
1144: i, pv->pv_ptep);
1.2 ryo 1145: i++;
1146: }
1147: }
1148: #endif /* PMAP_PV_DEBUG & DDB */
1149:
1150: static int
1.62 ryo 1151: _pmap_enter_pv(struct pmap_page *pp, struct pmap *pm, struct pv_entry **pvp,
1.35 ryo 1152: vaddr_t va, pt_entry_t *ptep, paddr_t pa, u_int flags)
1.2 ryo 1153: {
1154: struct pv_entry *pv;
1155:
1156: UVMHIST_FUNC(__func__);
1157: UVMHIST_CALLED(pmaphist);
1158:
1.62 ryo 1159: UVMHIST_LOG(pmaphist, "pp=%p, pm=%p, va=%llx, pa=%llx", pp, pm, va, pa);
1.2 ryo 1160: UVMHIST_LOG(pmaphist, "ptep=%p, flags=%08x", ptep, flags, 0, 0);
1161:
1.78 ad 1162: KASSERT(mutex_owned(&pp->pp_pvlock));
1163: KASSERT(trunc_page(va) == va);
1.2 ryo 1164:
1.78 ad 1165: /*
1166: * mapping cannot be already registered at this VA.
1167: */
1168: if (pp->pp_pv.pv_pmap == NULL) {
1169: /*
1170: * claim pv_entry embedded in pmap_page.
1171: * take care not to wipe out acc/mod flags.
1172: */
1173: pv = &pp->pp_pv;
1174: pv->pv_va = (pv->pv_va & (PAGE_SIZE - 1)) | va;
1175: } else {
1.12 ryo 1176: /*
1177: * create and link new pv.
1178: * pv is already allocated at beginning of _pmap_enter().
1179: */
1180: pv = *pvp;
1.2 ryo 1181: if (pv == NULL)
1182: return ENOMEM;
1.12 ryo 1183: *pvp = NULL;
1.78 ad 1184: pv->pv_next = pp->pp_pv.pv_next;
1185: pp->pp_pv.pv_next = pv;
1.2 ryo 1186: pv->pv_va = va;
1.78 ad 1187: }
1188: pv->pv_pmap = pm;
1189: pv->pv_ptep = ptep;
1190: PMAP_COUNT(pv_enter);
1.2 ryo 1191:
1192: #ifdef PMAP_PV_DEBUG
1.78 ad 1193: printf("pv %p alias added va=%016lx -> pa=%016lx\n", pv, va, pa);
1194: pv_dump(pp, printf);
1.2 ryo 1195: #endif
1.35 ryo 1196:
1.1 matt 1197: return 0;
1198: }
1199:
1200: void
1.2 ryo 1201: pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1.1 matt 1202: {
1.2 ryo 1203:
1204: _pmap_enter(pmap_kernel(), va, pa, prot, flags | PMAP_WIRED, true);
1.1 matt 1205: }
1206:
1207: void
1.2 ryo 1208: pmap_kremove(vaddr_t va, vsize_t size)
1.1 matt 1209: {
1.2 ryo 1210: struct pmap *kpm = pmap_kernel();
1211:
1212: UVMHIST_FUNC(__func__);
1213: UVMHIST_CALLED(pmaphist);
1214:
1215: UVMHIST_LOG(pmaphist, "va=%llx, size=%llx", va, size, 0, 0);
1216:
1217: KDASSERT((va & PGOFSET) == 0);
1218: KDASSERT((size & PGOFSET) == 0);
1219:
1.28 ryo 1220: KDASSERT(!IN_KSEG_ADDR(va));
1221: KDASSERT(IN_RANGE(va, VM_MIN_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS));
1.2 ryo 1222:
1.35 ryo 1223: pm_lock(kpm);
1224: _pmap_remove(kpm, va, va + size, true, NULL);
1225: pm_unlock(kpm);
1.2 ryo 1226: }
1227:
1228: static void
1.62 ryo 1229: _pmap_protect_pv(struct pmap_page *pp, struct pv_entry *pv, vm_prot_t prot)
1.2 ryo 1230: {
1231: pt_entry_t *ptep, pte;
1232: vm_prot_t pteprot;
1233: uint32_t mdattr;
1.18 ryo 1234: const bool user = (pv->pv_pmap != pmap_kernel());
1.2 ryo 1235:
1236: UVMHIST_FUNC(__func__);
1237: UVMHIST_CALLED(pmaphist);
1238:
1.62 ryo 1239: UVMHIST_LOG(pmaphist, "pp=%p, pv=%p, prot=%08x", pp, pv, prot, 0);
1.78 ad 1240: KASSERT(mutex_owned(&pv->pv_pmap->pm_lock));
1.2 ryo 1241:
1242: /* get prot mask from referenced/modified */
1.78 ad 1243: mdattr = pp->pp_pv.pv_va & (VM_PROT_READ | VM_PROT_WRITE);
1.2 ryo 1244: ptep = pv->pv_ptep;
1245: pte = *ptep;
1246:
1247: /* get prot mask from pte */
1248: pteprot = 0;
1.3 ryo 1249: if (pte & LX_BLKPAG_AF)
1.2 ryo 1250: pteprot |= VM_PROT_READ;
1.3 ryo 1251: if ((pte & LX_BLKPAG_AP) == LX_BLKPAG_AP_RW)
1.2 ryo 1252: pteprot |= VM_PROT_WRITE;
1.18 ryo 1253: if (l3pte_executable(pte, user))
1.2 ryo 1254: pteprot |= VM_PROT_EXECUTE;
1255:
1256: /* new prot = prot & pteprot & mdattr */
1.18 ryo 1257: pte = _pmap_pte_adjust_prot(pte, prot & pteprot, mdattr, user);
1.2 ryo 1258: atomic_swap_64(ptep, pte);
1.78 ad 1259: AARCH64_TLBI_BY_ASID_VA(pv->pv_pmap->pm_asid, trunc_page(pv->pv_va),
1260: true);
1.1 matt 1261: }
1262:
1263: void
1264: pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
1265: {
1.35 ryo 1266: pt_entry_t *ptep = NULL, pte;
1.2 ryo 1267: vaddr_t va;
1.35 ryo 1268: vsize_t blocksize = 0;
1.18 ryo 1269: const bool user = (pm != pmap_kernel());
1.2 ryo 1270:
1271: KASSERT((prot & VM_PROT_READ) || !(prot & VM_PROT_WRITE));
1272:
1273: UVMHIST_FUNC(__func__);
1274: UVMHIST_CALLED(pmaphist);
1275:
1276: UVMHIST_LOG(pmaphist, "pm=%p, sva=%016lx, eva=%016lx, prot=%08x",
1277: pm, sva, eva, prot);
1278:
1.28 ryo 1279: KASSERT_PM_ADDR(pm, sva);
1.2 ryo 1280: KASSERT(!IN_KSEG_ADDR(sva));
1281:
1.61 ryo 1282: /* PROT_EXEC requires implicit PROT_READ */
1283: if (prot & VM_PROT_EXECUTE)
1284: prot |= VM_PROT_READ;
1285:
1.2 ryo 1286: if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
1287: PMAP_COUNT(protect_remove_fallback);
1288: pmap_remove(pm, sva, eva);
1289: return;
1290: }
1291: PMAP_COUNT(protect);
1292:
1293: KDASSERT((sva & PAGE_MASK) == 0);
1294: KDASSERT((eva & PAGE_MASK) == 0);
1295:
1.11 ryo 1296: pm_lock(pm);
1.2 ryo 1297:
1.35 ryo 1298: for (va = sva; va < eva; va = (va + blocksize) & ~(blocksize - 1)) {
1.13 ryo 1299: #ifdef UVMHIST
1300: pt_entry_t opte;
1301: #endif
1.2 ryo 1302: struct vm_page *pg;
1.62 ryo 1303: struct pmap_page *pp;
1.2 ryo 1304: paddr_t pa;
1305: uint32_t mdattr;
1306: bool executable;
1307:
1.35 ryo 1308: /* va is belong to the same L3 table as before? */
1309: if ((blocksize == L3_SIZE) && ((va & L3INDEXMASK) != 0))
1310: ptep++;
1311: else
1312: ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
1.2 ryo 1313:
1314: pte = *ptep;
1.35 ryo 1315: if (!lxpde_valid(pte)) {
1.2 ryo 1316: PMAP_COUNT(protect_none);
1317: continue;
1318: }
1319:
1.35 ryo 1320: pa = lxpde_pa(pte);
1.2 ryo 1321: pg = PHYS_TO_VM_PAGE(pa);
1.62 ryo 1322: if (pg != NULL) {
1323: pp = VM_PAGE_TO_PP(pg);
1324: PMAP_COUNT(protect_managed);
1325: } else {
1.63 ryo 1326: #ifdef __HAVE_PMAP_PV_TRACK
1327: pp = pmap_pv_tracked(pa);
1328: #ifdef PMAPCOUNTERS
1329: if (pp != NULL)
1330: PMAP_COUNT(protect_pvmanaged);
1331: else
1332: PMAP_COUNT(protect_unmanaged);
1333: #endif
1334: #else
1.62 ryo 1335: pp = NULL;
1336: PMAP_COUNT(protect_unmanaged);
1.63 ryo 1337: #endif /* __HAVE_PMAP_PV_TRACK */
1.62 ryo 1338: }
1.2 ryo 1339:
1.62 ryo 1340: if (pp != NULL) {
1.3 ryo 1341: /* get prot mask from referenced/modified */
1.78 ad 1342: mdattr = pp->pp_pv.pv_va &
1343: (VM_PROT_READ | VM_PROT_WRITE);
1.2 ryo 1344: } else {
1345: /* unmanaged page */
1346: mdattr = VM_PROT_ALL;
1347: }
1348:
1.13 ryo 1349: #ifdef UVMHIST
1350: opte = pte;
1351: #endif
1.18 ryo 1352: executable = l3pte_executable(pte, user);
1353: pte = _pmap_pte_adjust_prot(pte, prot, mdattr, user);
1.13 ryo 1354:
1355: if (!executable && (prot & VM_PROT_EXECUTE)) {
1356: /* non-exec -> exec */
1.15 ryo 1357: UVMHIST_LOG(pmaphist, "icache_sync: "
1358: "pm=%p, va=%016lx, pte: %016lx -> %016lx",
1.13 ryo 1359: pm, va, opte, pte);
1.76 ryo 1360: if (!l3pte_readable(pte)) {
1.19 ryo 1361: PTE_ICACHE_SYNC_PAGE(pte, ptep, pm, va, true);
1.15 ryo 1362: atomic_swap_64(ptep, pte);
1.19 ryo 1363: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
1.15 ryo 1364: } else {
1365: atomic_swap_64(ptep, pte);
1.19 ryo 1366: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
1.15 ryo 1367: cpu_icache_sync_range(va, PAGE_SIZE);
1368: }
1369: } else {
1370: atomic_swap_64(ptep, pte);
1.19 ryo 1371: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
1.15 ryo 1372: }
1.2 ryo 1373: }
1374:
1.11 ryo 1375: pm_unlock(pm);
1.1 matt 1376: }
1377:
1378: void
1.2 ryo 1379: pmap_activate(struct lwp *l)
1380: {
1381: struct pmap *pm = l->l_proc->p_vmspace->vm_map.pmap;
1.47 jmcneill 1382: uint64_t ttbr0, tcr;
1.2 ryo 1383:
1384: UVMHIST_FUNC(__func__);
1385: UVMHIST_CALLED(pmaphist);
1386:
1387: if (pm == pmap_kernel())
1388: return;
1389: if (l != curlwp)
1390: return;
1391:
1392: KASSERT(pm->pm_l0table != NULL);
1393:
1394: UVMHIST_LOG(pmaphist, "lwp=%p (pid=%d)", l, l->l_proc->p_pid, 0, 0);
1395:
1.47 jmcneill 1396: /* Disable translation table walks using TTBR0 */
1397: tcr = reg_tcr_el1_read();
1398: reg_tcr_el1_write(tcr | TCR_EPD0);
1399: __asm __volatile("isb" ::: "memory");
1400:
1.2 ryo 1401: /* XXX */
1402: CTASSERT(PID_MAX <= 65535); /* 16bit ASID */
1403: if (pm->pm_asid == -1)
1404: pm->pm_asid = l->l_proc->p_pid;
1405:
1406: ttbr0 = ((uint64_t)pm->pm_asid << 48) | pm->pm_l0table_pa;
1.34 ryo 1407: cpu_set_ttbr0(ttbr0);
1.2 ryo 1408:
1.47 jmcneill 1409: /* Re-enable translation table walks using TTBR0 */
1410: tcr = reg_tcr_el1_read();
1411: reg_tcr_el1_write(tcr & ~TCR_EPD0);
1412: __asm __volatile("isb" ::: "memory");
1413:
1.2 ryo 1414: pm->pm_activated = true;
1415:
1416: PMAP_COUNT(activate);
1417: }
1418:
1419: void
1420: pmap_deactivate(struct lwp *l)
1.1 matt 1421: {
1.2 ryo 1422: struct pmap *pm = l->l_proc->p_vmspace->vm_map.pmap;
1.47 jmcneill 1423: uint64_t tcr;
1.2 ryo 1424:
1425: UVMHIST_FUNC(__func__);
1426: UVMHIST_CALLED(pmaphist);
1427:
1428: if (pm == pmap_kernel())
1429: return;
1430:
1431: UVMHIST_LOG(pmaphist, "lwp=%p, asid=%d", l, pm->pm_asid, 0, 0);
1432:
1.47 jmcneill 1433: /* Disable translation table walks using TTBR0 */
1434: tcr = reg_tcr_el1_read();
1435: reg_tcr_el1_write(tcr | TCR_EPD0);
1436: __asm __volatile("isb" ::: "memory");
1437:
1.2 ryo 1438: /* XXX */
1439: pm->pm_activated = false;
1440:
1441: PMAP_COUNT(deactivate);
1.1 matt 1442: }
1443:
1.2 ryo 1444: struct pmap *
1445: pmap_create(void)
1.1 matt 1446: {
1.2 ryo 1447: struct pmap *pm;
1448:
1449: UVMHIST_FUNC(__func__);
1450: UVMHIST_CALLED(pmaphist);
1451:
1452: pm = pool_cache_get(&_pmap_cache, PR_WAITOK);
1453: memset(pm, 0, sizeof(*pm));
1454: pm->pm_refcnt = 1;
1.40 ryo 1455: pm->pm_idlepdp = 0;
1.2 ryo 1456: pm->pm_asid = -1;
1.64 ryo 1457: LIST_INIT(&pm->pm_vmlist);
1.6 ryo 1458: mutex_init(&pm->pm_lock, MUTEX_DEFAULT, IPL_VM);
1.39 ryo 1459:
1.40 ryo 1460: pm->pm_l0table_pa = pmap_alloc_pdp(pm, NULL, 0, true);
1.36 ryo 1461: KASSERT(pm->pm_l0table_pa != POOL_PADDR_INVALID);
1462: pm->pm_l0table = (pd_entry_t *)AARCH64_PA_TO_KVA(pm->pm_l0table_pa);
1.2 ryo 1463: KASSERT(((vaddr_t)pm->pm_l0table & (PAGE_SIZE - 1)) == 0);
1464:
1465: UVMHIST_LOG(pmaphist, "pm=%p, pm_l0table=%016lx, pm_l0table_pa=%016lx",
1466: pm, pm->pm_l0table, pm->pm_l0table_pa, 0);
1467:
1468: PMAP_COUNT(create);
1469: return pm;
1.1 matt 1470: }
1471:
1472: void
1.2 ryo 1473: pmap_destroy(struct pmap *pm)
1474: {
1475: unsigned int refcnt;
1476:
1477: UVMHIST_FUNC(__func__);
1478: UVMHIST_CALLED(pmaphist);
1479:
1480: UVMHIST_LOG(pmaphist,
1481: "pm=%p, pm_l0table=%016lx, pm_l0table_pa=%016lx, refcnt=%d",
1482: pm, pm->pm_l0table, pm->pm_l0table_pa, pm->pm_refcnt);
1483:
1484: if (pm == NULL)
1485: return;
1486:
1487: if (pm == pmap_kernel())
1488: panic("cannot destroy kernel pmap");
1489:
1490: refcnt = atomic_dec_uint_nv(&pm->pm_refcnt);
1491: if (refcnt > 0)
1492: return;
1493:
1494: aarch64_tlbi_by_asid(pm->pm_asid);
1495:
1496: _pmap_free_pdp_all(pm);
1497: mutex_destroy(&pm->pm_lock);
1.35 ryo 1498:
1.2 ryo 1499: pool_cache_put(&_pmap_cache, pm);
1500:
1501: PMAP_COUNT(destroy);
1502: }
1503:
1.36 ryo 1504: static inline void
1505: _pmap_pdp_setparent(struct pmap *pm, struct vm_page *pg, pt_entry_t *ptep)
1506: {
1.78 ad 1507:
1508: if ((pm != pmap_kernel()) && (pg != NULL)) {
1509: KASSERT(mutex_owned(&pm->pm_lock));
1.36 ryo 1510: VM_PAGE_TO_MD(pg)->mdpg_ptep_parent = ptep;
1.78 ad 1511: }
1.36 ryo 1512: }
1513:
1514: /*
1515: * increment reference counter of the page descriptor page.
1516: * the reference counter should be equal to
1517: * 1 + num of valid entries the page has.
1518: */
1519: static inline void
1520: _pmap_pdp_addref(struct pmap *pm, paddr_t pdppa, struct vm_page *pdppg_hint)
1521: {
1522: struct vm_page *pg;
1523:
1524: /* kernel L0-L3 page will be never freed */
1525: if (pm == pmap_kernel())
1526: return;
1.78 ad 1527:
1528: KASSERT(mutex_owned(&pm->pm_lock));
1529:
1.36 ryo 1530: /* no need for L0 page */
1531: if (pm->pm_l0table_pa == pdppa)
1532: return;
1533:
1534: pg = pdppg_hint;
1535: if (pg == NULL)
1536: pg = PHYS_TO_VM_PAGE(pdppa);
1537: KASSERT(pg != NULL);
1538:
1.78 ad 1539: pg->wire_count++;
1.36 ryo 1540:
1541: KASSERTMSG(pg->wire_count <= (Ln_ENTRIES + 1),
1542: "pg=%p, wire_count=%d", pg, pg->wire_count);
1543: }
1544:
1545: /*
1546: * decrement reference counter of the page descriptr page.
1547: * if reference counter is 1(=empty), pages will be freed, and return true.
1548: * otherwise return false.
1549: * kernel page, or L0 page descriptor page will be never freed.
1550: */
1551: static bool
1552: _pmap_pdp_delref(struct pmap *pm, paddr_t pdppa, bool do_free_pdp)
1553: {
1554: struct vm_page *pg;
1555: bool removed;
1556: uint16_t wirecount;
1557:
1558: /* kernel L0-L3 page will be never freed */
1559: if (pm == pmap_kernel())
1560: return false;
1.78 ad 1561:
1562: KASSERT(mutex_owned(&pm->pm_lock));
1563:
1.36 ryo 1564: /* no need for L0 page */
1565: if (pm->pm_l0table_pa == pdppa)
1566: return false;
1567:
1568: pg = PHYS_TO_VM_PAGE(pdppa);
1569: KASSERT(pg != NULL);
1570:
1.78 ad 1571: wirecount = --pg->wire_count;
1.36 ryo 1572:
1.40 ryo 1573: if (!do_free_pdp) {
1574: /*
1575: * pm_idlepdp is counted by only pmap_page_protect() with
1576: * VM_PROT_NONE. it is not correct because without considering
1577: * pmap_enter(), but useful hint to just sweep.
1578: */
1579: if (wirecount == 1)
1.78 ad 1580: pm->pm_idlepdp++;
1.36 ryo 1581: return false;
1.40 ryo 1582: }
1.36 ryo 1583:
1584: /* if no reference, free pdp */
1585: removed = false;
1586: while (wirecount == 1) {
1.41 mrg 1587: pd_entry_t *ptep_in_parent, opte __diagused;
1.36 ryo 1588: ptep_in_parent = VM_PAGE_TO_MD(pg)->mdpg_ptep_parent;
1589: if (ptep_in_parent == NULL) {
1590: /* no parent */
1591: pmap_free_pdp(pm, pg);
1592: removed = true;
1593: break;
1594: }
1595:
1596: /* unlink from parent */
1597: opte = atomic_swap_64(ptep_in_parent, 0);
1598: KASSERT(lxpde_valid(opte));
1.53 skrll 1599: wirecount = atomic_add_32_nv(&pg->wire_count, -1); /* 1 -> 0 */
1.36 ryo 1600: KASSERT(wirecount == 0);
1601: pmap_free_pdp(pm, pg);
1602: removed = true;
1603:
1604: /* L3->L2->L1. no need for L0 */
1605: pdppa = AARCH64_KVA_TO_PA(trunc_page((vaddr_t)ptep_in_parent));
1606: if (pdppa == pm->pm_l0table_pa)
1607: break;
1608:
1609: pg = PHYS_TO_VM_PAGE(pdppa);
1610: KASSERT(pg != NULL);
1611: KASSERTMSG(pg->wire_count >= 1,
1612: "wire_count=%d", pg->wire_count);
1613: /* decrement wire_count of parent */
1.53 skrll 1614: wirecount = atomic_add_32_nv(&pg->wire_count, -1);
1.36 ryo 1615: KASSERTMSG(pg->wire_count <= (Ln_ENTRIES + 1),
1616: "pm=%p[%d], pg=%p, wire_count=%d",
1617: pm, pm->pm_asid, pg, pg->wire_count);
1618: }
1619:
1620: return removed;
1621: }
1622:
1.2 ryo 1623: static int
1624: _pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot,
1625: u_int flags, bool kenter)
1626: {
1.62 ryo 1627: struct vm_page *pdppg, *pdppg0;
1628: struct pmap_page *pp, *opp, *pps[2];
1.14 ryo 1629: struct pv_entry *spv, *opv = NULL;
1.2 ryo 1630: pd_entry_t pde;
1.39 ryo 1631: pt_entry_t attr, pte, opte, *ptep;
1.2 ryo 1632: pd_entry_t *l0, *l1, *l2, *l3;
1.36 ryo 1633: paddr_t pdppa, pdppa0;
1.2 ryo 1634: uint32_t mdattr;
1635: unsigned int idx;
1636: int error = 0;
1637: const bool user = (pm != pmap_kernel());
1.77 ad 1638: bool need_sync_icache, need_enter_pv;
1.19 ryo 1639: bool l3only = true;
1.2 ryo 1640:
1641: UVMHIST_FUNC(__func__);
1642: UVMHIST_CALLED(pmaphist);
1643:
1644: UVMHIST_LOG(pmaphist, "pm=%p, kentermode=%d", pm, kenter, 0, 0);
1645: UVMHIST_LOG(pmaphist, "va=%016lx, pa=%016lx, prot=%08x, flags=%08x",
1646: va, pa, prot, flags);
1647:
1.28 ryo 1648: KASSERT_PM_ADDR(pm, va);
1649: KASSERT(!IN_KSEG_ADDR(va));
1.2 ryo 1650:
1651: #ifdef PMAPCOUNTERS
1652: PMAP_COUNT(mappings);
1653: if (_pmap_color(va) == _pmap_color(pa)) {
1654: if (user) {
1655: PMAP_COUNT(user_mappings);
1656: } else {
1657: PMAP_COUNT(kern_mappings);
1658: }
1659: } else if (flags & PMAP_WIRED) {
1660: if (user) {
1661: PMAP_COUNT(user_mappings_bad_wired);
1662: } else {
1663: PMAP_COUNT(kern_mappings_bad_wired);
1664: }
1665: } else {
1666: if (user) {
1667: PMAP_COUNT(user_mappings_bad);
1668: } else {
1669: PMAP_COUNT(kern_mappings_bad);
1670: }
1671: }
1672: #endif
1673:
1.62 ryo 1674: if (kenter) {
1675: pp = NULL;
1676: } else {
1677: struct vm_page *pg = PHYS_TO_VM_PAGE(pa);
1678: if (pg != NULL) {
1679: pp = VM_PAGE_TO_PP(pg);
1680: PMAP_COUNT(managed_mappings);
1681: } else {
1.63 ryo 1682: #ifdef __HAVE_PMAP_PV_TRACK
1683: pp = pmap_pv_tracked(pa);
1684: #ifdef PMAPCOUNTERS
1685: if (pp != NULL)
1686: PMAP_COUNT(pvmanaged_mappings);
1687: else
1688: PMAP_COUNT(unmanaged_mappings);
1689: #endif
1690: #else
1.62 ryo 1691: pp = NULL;
1692: PMAP_COUNT(unmanaged_mappings);
1.63 ryo 1693: #endif /* __HAVE_PMAP_PV_TRACK */
1.62 ryo 1694: }
1695: }
1.3 ryo 1696:
1.62 ryo 1697: if (pp != NULL) {
1.12 ryo 1698: /*
1699: * allocate pv in advance of pm_lock() to avoid locking myself.
1700: * pool_cache_get() may call pmap_kenter() internally.
1701: */
1702: spv = pool_cache_get(&_pmap_pv_pool, PR_NOWAIT);
1.77 ad 1703: need_enter_pv = true;
1.12 ryo 1704: } else {
1705: spv = NULL;
1.77 ad 1706: need_enter_pv = false;
1.2 ryo 1707: }
1708:
1.12 ryo 1709: pm_lock(pm);
1.2 ryo 1710:
1.40 ryo 1711: if (pm->pm_idlepdp >= PDPSWEEP_TRIGGER) {
1712: if (_pmap_sweep_pdp(pm) != 0) {
1713: /* several L1-L3 page table pages have been freed */
1714: aarch64_tlbi_by_asid(pm->pm_asid);
1715: }
1716: }
1717:
1.2 ryo 1718: /*
1719: * traverse L0 -> L1 -> L2 -> L3 table with growing pdp if needed.
1720: */
1721: l0 = pm->pm_l0table;
1722:
1723: idx = l0pde_index(va);
1724: pde = l0[idx];
1725: if (!l0pde_valid(pde)) {
1.36 ryo 1726: /* no need to increment L0 occupancy. L0 page never freed */
1.40 ryo 1727: pdppa = pmap_alloc_pdp(pm, &pdppg, flags, false); /* L1 pdp */
1.36 ryo 1728: if (pdppa == POOL_PADDR_INVALID) {
1729: if (flags & PMAP_CANFAIL) {
1730: error = ENOMEM;
1.39 ryo 1731: goto fail0;
1.36 ryo 1732: }
1.39 ryo 1733: pm_unlock(pm);
1.36 ryo 1734: panic("%s: cannot allocate L1 table", __func__);
1735: }
1.2 ryo 1736: atomic_swap_64(&l0[idx], pdppa | L0_TABLE);
1.36 ryo 1737: _pmap_pdp_setparent(pm, pdppg, &l0[idx]);
1.19 ryo 1738: l3only = false;
1.2 ryo 1739: } else {
1740: pdppa = l0pde_pa(pde);
1.36 ryo 1741: pdppg = NULL;
1.2 ryo 1742: }
1.9 christos 1743: l1 = (void *)AARCH64_PA_TO_KVA(pdppa);
1.2 ryo 1744:
1745: idx = l1pde_index(va);
1746: pde = l1[idx];
1747: if (!l1pde_valid(pde)) {
1.36 ryo 1748: pdppa0 = pdppa;
1749: pdppg0 = pdppg;
1.40 ryo 1750: pdppa = pmap_alloc_pdp(pm, &pdppg, flags, false); /* L2 pdp */
1.36 ryo 1751: if (pdppa == POOL_PADDR_INVALID) {
1752: if (flags & PMAP_CANFAIL) {
1753: error = ENOMEM;
1.39 ryo 1754: goto fail0;
1.36 ryo 1755: }
1.39 ryo 1756: pm_unlock(pm);
1.36 ryo 1757: panic("%s: cannot allocate L2 table", __func__);
1758: }
1.2 ryo 1759: atomic_swap_64(&l1[idx], pdppa | L1_TABLE);
1.36 ryo 1760: _pmap_pdp_addref(pm, pdppa0, pdppg0); /* L1 occupancy++ */
1761: _pmap_pdp_setparent(pm, pdppg, &l1[idx]);
1.19 ryo 1762: l3only = false;
1.2 ryo 1763: } else {
1764: pdppa = l1pde_pa(pde);
1.36 ryo 1765: pdppg = NULL;
1.2 ryo 1766: }
1.9 christos 1767: l2 = (void *)AARCH64_PA_TO_KVA(pdppa);
1.2 ryo 1768:
1769: idx = l2pde_index(va);
1770: pde = l2[idx];
1771: if (!l2pde_valid(pde)) {
1.36 ryo 1772: pdppa0 = pdppa;
1773: pdppg0 = pdppg;
1.40 ryo 1774: pdppa = pmap_alloc_pdp(pm, &pdppg, flags, false); /* L3 pdp */
1.36 ryo 1775: if (pdppa == POOL_PADDR_INVALID) {
1776: if (flags & PMAP_CANFAIL) {
1777: error = ENOMEM;
1.39 ryo 1778: goto fail0;
1.36 ryo 1779: }
1.39 ryo 1780: pm_unlock(pm);
1.36 ryo 1781: panic("%s: cannot allocate L3 table", __func__);
1782: }
1.2 ryo 1783: atomic_swap_64(&l2[idx], pdppa | L2_TABLE);
1.36 ryo 1784: _pmap_pdp_addref(pm, pdppa0, pdppg0); /* L2 occupancy++ */
1785: _pmap_pdp_setparent(pm, pdppg, &l2[idx]);
1.19 ryo 1786: l3only = false;
1.2 ryo 1787: } else {
1788: pdppa = l2pde_pa(pde);
1.36 ryo 1789: pdppg = NULL;
1.2 ryo 1790: }
1.9 christos 1791: l3 = (void *)AARCH64_PA_TO_KVA(pdppa);
1.2 ryo 1792:
1793: idx = l3pte_index(va);
1794: ptep = &l3[idx]; /* as PTE */
1795:
1.39 ryo 1796: opte = atomic_swap_64(ptep, 0);
1.32 ryo 1797: need_sync_icache = (prot & VM_PROT_EXECUTE);
1.2 ryo 1798:
1.62 ryo 1799: /* for lock ordering for old page and new page */
1800: pps[0] = pp;
1801: pps[1] = NULL;
1.39 ryo 1802:
1803: /* remap? */
1804: if (l3pte_valid(opte)) {
1805: bool need_remove_pv;
1806:
1.2 ryo 1807: KASSERT(!kenter); /* pmap_kenter_pa() cannot override */
1.39 ryo 1808: if (opte & LX_BLKPAG_OS_WIRED) {
1809: PMSTAT_DEC_WIRED_COUNT(pm);
1810: }
1811: PMSTAT_DEC_RESIDENT_COUNT(pm);
1.77 ad 1812: #ifdef PMAPCOUNTERS
1813: PMAP_COUNT(remappings);
1.39 ryo 1814: if (user) {
1815: PMAP_COUNT(user_mappings_changed);
1816: } else {
1817: PMAP_COUNT(kern_mappings_changed);
1818: }
1819: #endif
1820: UVMHIST_LOG(pmaphist,
1821: "va=%016lx has already mapped."
1822: " old-pa=%016lx new-pa=%016lx, old-pte=%016llx\n",
1823: va, l3pte_pa(opte), pa, opte);
1824:
1825: if (pa == l3pte_pa(opte)) {
1826: /* old and new pte have same pa, no need to update pv */
1.62 ryo 1827: need_remove_pv = (pp == NULL);
1.77 ad 1828: need_enter_pv = false;
1.39 ryo 1829: if (need_sync_icache && l3pte_executable(opte, user))
1.32 ryo 1830: need_sync_icache = false;
1831: } else {
1.39 ryo 1832: need_remove_pv = true;
1833: }
1.40 ryo 1834:
1835: if (need_remove_pv &&
1.62 ryo 1836: ((opp = phys_to_pp(l3pte_pa(opte))) != NULL)) {
1837: /*
1838: * need to lock both pp and opp(old pp)
1839: * against deadlock, and 'pp' maybe NULL.
1840: */
1841: if (pp < opp) {
1842: pps[0] = pp;
1843: pps[1] = opp;
1.40 ryo 1844: } else {
1.62 ryo 1845: pps[0] = opp;
1846: pps[1] = pp;
1.2 ryo 1847: }
1.62 ryo 1848: if (pps[0] != NULL)
1849: pmap_pv_lock(pps[0]);
1850: if (pps[1] != NULL)
1851: pmap_pv_lock(pps[1]);
1852: opv = _pmap_remove_pv(opp, pm, va, opte);
1.40 ryo 1853: } else {
1.62 ryo 1854: if (pp != NULL)
1855: pmap_pv_lock(pp);
1.36 ryo 1856: }
1.40 ryo 1857: } else {
1.62 ryo 1858: if (pp != NULL)
1859: pmap_pv_lock(pp);
1.2 ryo 1860: }
1861:
1.40 ryo 1862: if (!l3pte_valid(opte))
1863: _pmap_pdp_addref(pm, pdppa, pdppg); /* L3 occupancy++ */
1864:
1.20 ryo 1865: /*
1866: * read permission is treated as an access permission internally.
1867: * require to add PROT_READ even if only PROT_WRITE or PROT_EXEC
1868: */
1.44 ryo 1869: if (prot & (VM_PROT_WRITE|VM_PROT_EXECUTE))
1.20 ryo 1870: prot |= VM_PROT_READ;
1.46 ryo 1871: if (flags & (VM_PROT_WRITE|VM_PROT_EXECUTE))
1872: flags |= VM_PROT_READ;
1.20 ryo 1873:
1.2 ryo 1874: mdattr = VM_PROT_READ | VM_PROT_WRITE;
1.77 ad 1875: if (need_enter_pv) {
1.62 ryo 1876: error = _pmap_enter_pv(pp, pm, &spv, va, ptep, pa, flags);
1.3 ryo 1877: if (error != 0) {
1.11 ryo 1878: /*
1879: * If pmap_enter() fails,
1880: * it must not leave behind an existing pmap entry.
1881: */
1.39 ryo 1882: if (lxpde_valid(opte)) {
1883: bool pdpremoved = _pmap_pdp_delref(pm,
1884: AARCH64_KVA_TO_PA(trunc_page(
1885: (vaddr_t)ptep)), true);
1886: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid,
1887: va, !pdpremoved);
1888: }
1.3 ryo 1889: PMAP_COUNT(pv_entry_cannotalloc);
1890: if (flags & PMAP_CANFAIL)
1.39 ryo 1891: goto fail1;
1.11 ryo 1892: panic("pmap_enter: failed to allocate pv_entry");
1.3 ryo 1893: }
1.39 ryo 1894: }
1.2 ryo 1895:
1.62 ryo 1896: if (pp != NULL) {
1.7 ryo 1897: /* update referenced/modified flags */
1.78 ad 1898: pp->pp_pv.pv_va |= (flags & (VM_PROT_READ | VM_PROT_WRITE));
1899: mdattr &= (uint32_t)pp->pp_pv.pv_va;
1.2 ryo 1900: }
1901:
1902: #ifdef PMAPCOUNTERS
1903: switch (flags & PMAP_CACHE_MASK) {
1904: case PMAP_NOCACHE:
1905: case PMAP_NOCACHE_OVR:
1906: PMAP_COUNT(uncached_mappings);
1907: break;
1908: }
1909: #endif
1910:
1.18 ryo 1911: attr = _pmap_pte_adjust_prot(L3_PAGE, prot, mdattr, user);
1.2 ryo 1912: attr = _pmap_pte_adjust_cacheflags(attr, flags);
1913: if (VM_MAXUSER_ADDRESS > va)
1914: attr |= LX_BLKPAG_APUSER;
1.3 ryo 1915: if (flags & PMAP_WIRED)
1916: attr |= LX_BLKPAG_OS_WIRED;
1.2 ryo 1917: #ifdef MULTIPROCESSOR
1918: attr |= LX_BLKPAG_SH_IS;
1919: #endif
1920:
1921: pte = pa | attr;
1.13 ryo 1922:
1.32 ryo 1923: if (need_sync_icache) {
1.15 ryo 1924: /* non-exec -> exec */
1925: UVMHIST_LOG(pmaphist,
1926: "icache_sync: pm=%p, va=%016lx, pte: %016lx -> %016lx",
1.13 ryo 1927: pm, va, opte, pte);
1.76 ryo 1928: if (!l3pte_readable(pte)) {
1.19 ryo 1929: PTE_ICACHE_SYNC_PAGE(pte, ptep, pm, va, l3only);
1.15 ryo 1930: atomic_swap_64(ptep, pte);
1.19 ryo 1931: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va ,true);
1.15 ryo 1932: } else {
1933: atomic_swap_64(ptep, pte);
1.19 ryo 1934: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, l3only);
1.15 ryo 1935: cpu_icache_sync_range(va, PAGE_SIZE);
1936: }
1937: } else {
1938: atomic_swap_64(ptep, pte);
1.19 ryo 1939: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, l3only);
1.15 ryo 1940: }
1.39 ryo 1941:
1.36 ryo 1942: if (pte & LX_BLKPAG_OS_WIRED) {
1943: PMSTAT_INC_WIRED_COUNT(pm);
1944: }
1945: PMSTAT_INC_RESIDENT_COUNT(pm);
1.2 ryo 1946:
1.39 ryo 1947: fail1:
1.62 ryo 1948: if (pps[1] != NULL)
1949: pmap_pv_unlock(pps[1]);
1950: if (pps[0] != NULL)
1951: pmap_pv_unlock(pps[0]);
1.39 ryo 1952: fail0:
1.12 ryo 1953: pm_unlock(pm);
1954:
1955: /* spare pv was not used. discard */
1956: if (spv != NULL)
1957: pool_cache_put(&_pmap_pv_pool, spv);
1958:
1.14 ryo 1959: if (opv != NULL)
1960: pool_cache_put(&_pmap_pv_pool, opv);
1961:
1.2 ryo 1962: return error;
1963: }
1964:
1965: int
1966: pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1.1 matt 1967: {
1.2 ryo 1968: return _pmap_enter(pm, va, pa, prot, flags, false);
1.1 matt 1969: }
1970:
1.68 ad 1971: bool
1.2 ryo 1972: pmap_remove_all(struct pmap *pm)
1.1 matt 1973: {
1.2 ryo 1974: /* nothing to do */
1.68 ad 1975: return false;
1.1 matt 1976: }
1977:
1.2 ryo 1978: static void
1.35 ryo 1979: _pmap_remove(struct pmap *pm, vaddr_t sva, vaddr_t eva, bool kremove,
1980: struct pv_entry **pvtofree)
1.1 matt 1981: {
1.35 ryo 1982: pt_entry_t pte, *ptep = NULL;
1.62 ryo 1983: struct pmap_page *pp;
1.35 ryo 1984: struct pv_entry *opv;
1.2 ryo 1985: paddr_t pa;
1.35 ryo 1986: vaddr_t va;
1987: vsize_t blocksize = 0;
1.36 ryo 1988: bool pdpremoved;
1.2 ryo 1989:
1990: UVMHIST_FUNC(__func__);
1991: UVMHIST_CALLED(pmaphist);
1992:
1.35 ryo 1993: UVMHIST_LOG(pmaphist, "pm=%p, sva=%016lx, eva=%016lx, kremovemode=%d",
1994: pm, sva, eva, kremove);
1.2 ryo 1995:
1.35 ryo 1996: for (va = sva; (va < eva) && (pm->pm_stats.resident_count != 0);
1997: va = (va + blocksize) & ~(blocksize - 1)) {
1998:
1999: /* va is belong to the same L3 table as before? */
2000: if ((blocksize == L3_SIZE) && ((va & L3INDEXMASK) != 0))
2001: ptep++;
2002: else
2003: ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
1.8 ryo 2004:
1.2 ryo 2005: pte = *ptep;
1.35 ryo 2006: if (!lxpde_valid(pte))
2007: continue;
1.2 ryo 2008:
1.35 ryo 2009: if (!kremove) {
2010: pa = lxpde_pa(pte);
1.62 ryo 2011: pp = phys_to_pp(pa);
2012: if (pp != NULL) {
1.39 ryo 2013:
1.62 ryo 2014: pmap_pv_lock(pp);
2015: opv = _pmap_remove_pv(pp, pm, va, pte);
2016: pmap_pv_unlock(pp);
1.35 ryo 2017: if (opv != NULL) {
2018: opv->pv_next = *pvtofree;
2019: *pvtofree = opv;
2020: }
2021: }
2022: }
1.2 ryo 2023:
1.36 ryo 2024: pte = atomic_swap_64(ptep, 0);
2025: if (!lxpde_valid(pte))
2026: continue;
2027:
2028: pdpremoved = _pmap_pdp_delref(pm,
2029: AARCH64_KVA_TO_PA(trunc_page((vaddr_t)ptep)), true);
2030: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, !pdpremoved);
1.3 ryo 2031:
1.36 ryo 2032: if (pdpremoved) {
2033: /*
2034: * this Ln page table page has been removed.
2035: * skip to next Ln table
2036: */
2037: blocksize *= Ln_ENTRIES;
2038: }
2039:
2040: if ((pte & LX_BLKPAG_OS_WIRED) != 0) {
2041: PMSTAT_DEC_WIRED_COUNT(pm);
2042: }
2043: PMSTAT_DEC_RESIDENT_COUNT(pm);
1.2 ryo 2044: }
1.1 matt 2045: }
2046:
2047: void
1.2 ryo 2048: pmap_remove(struct pmap *pm, vaddr_t sva, vaddr_t eva)
1.1 matt 2049: {
1.35 ryo 2050: struct pv_entry *pvtofree = NULL;
2051: struct pv_entry *pv, *pvtmp;
1.2 ryo 2052:
1.28 ryo 2053: KASSERT_PM_ADDR(pm, sva);
1.2 ryo 2054: KASSERT(!IN_KSEG_ADDR(sva));
2055:
1.35 ryo 2056: pm_lock(pm);
2057: _pmap_remove(pm, sva, eva, false, &pvtofree);
2058: pm_unlock(pm);
2059:
2060: for (pv = pvtofree; pv != NULL; pv = pvtmp) {
2061: pvtmp = pv->pv_next;
2062: pool_cache_put(&_pmap_pv_pool, pv);
2063: }
1.1 matt 2064: }
2065:
1.63 ryo 2066: static void
2067: pmap_page_remove(struct pmap_page *pp, vm_prot_t prot)
1.1 matt 2068: {
1.2 ryo 2069: struct pv_entry *pv, *pvtmp;
1.63 ryo 2070: struct pv_entry *pvtofree = NULL;
1.78 ad 2071: struct pmap *pm;
1.3 ryo 2072: pt_entry_t opte;
1.2 ryo 2073:
1.78 ad 2074: /* remove all pages reference to this physical page */
2075: pmap_pv_lock(pp);
2076: for (pv = &pp->pp_pv; pv != NULL;) {
2077: if ((pm = pv->pv_pmap) == NULL) {
2078: KASSERT(pv == &pp->pp_pv);
2079: pv = pp->pp_pv.pv_next;
2080: continue;
2081: }
2082: if (!pm_reverse_lock(pm, pp)) {
2083: /* now retry */
2084: pv = &pp->pp_pv;
2085: continue;
2086: }
2087: opte = atomic_swap_64(pv->pv_ptep, 0);
2088: if (lxpde_valid(opte)) {
2089: _pmap_pdp_delref(pv->pv_pmap,
2090: AARCH64_KVA_TO_PA(trunc_page(
2091: (vaddr_t)pv->pv_ptep)), false);
2092: AARCH64_TLBI_BY_ASID_VA(pv->pv_pmap->pm_asid,
2093: trunc_page(pv->pv_va), true);
1.2 ryo 2094:
1.78 ad 2095: if ((opte & LX_BLKPAG_OS_WIRED) != 0) {
2096: PMSTAT_DEC_WIRED_COUNT(pv->pv_pmap);
1.36 ryo 2097: }
1.78 ad 2098: PMSTAT_DEC_RESIDENT_COUNT(pv->pv_pmap);
2099: }
2100: pvtmp = _pmap_remove_pv(pp, pm, trunc_page(pv->pv_va), opte);
2101: if (pvtmp == NULL) {
2102: KASSERT(pv == &pp->pp_pv);
2103: } else {
2104: KASSERT(pv == pvtmp);
2105: pp->pp_pv.pv_next = pv->pv_next;
1.35 ryo 2106: pv->pv_next = pvtofree;
2107: pvtofree = pv;
1.2 ryo 2108: }
1.78 ad 2109: pm_unlock(pm);
2110: pv = pp->pp_pv.pv_next;
2111: }
2112: pmap_pv_unlock(pp);
1.2 ryo 2113:
1.78 ad 2114: for (pv = pvtofree; pv != NULL; pv = pvtmp) {
2115: pvtmp = pv->pv_next;
2116: pool_cache_put(&_pmap_pv_pool, pv);
2117: }
1.63 ryo 2118: }
2119:
2120: #ifdef __HAVE_PMAP_PV_TRACK
2121: void
2122: pmap_pv_protect(paddr_t pa, vm_prot_t prot)
2123: {
2124: struct pmap_page *pp;
2125:
2126: UVMHIST_FUNC(__func__);
2127: UVMHIST_CALLED(pmaphist);
2128:
2129: UVMHIST_LOG(pmaphist, "pa=%016lx, prot=%08x",
2130: pa, prot, 0, 0);
2131:
2132: pp = pmap_pv_tracked(pa);
2133: if (pp == NULL)
2134: panic("pmap_pv_protect: page not pv-tracked: %#" PRIxPADDR, pa);
2135:
2136: KASSERT(prot == VM_PROT_NONE);
2137: pmap_page_remove(pp, prot);
2138: }
2139: #endif
2140:
2141: void
2142: pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
2143: {
2144: struct pv_entry *pv;
2145: struct pmap_page *pp;
1.78 ad 2146: struct pmap *pm;
1.63 ryo 2147:
2148: KASSERT((prot & VM_PROT_READ) || !(prot & VM_PROT_WRITE));
2149:
2150: UVMHIST_FUNC(__func__);
2151: UVMHIST_CALLED(pmaphist);
2152:
2153: pp = VM_PAGE_TO_PP(pg);
2154:
2155: UVMHIST_LOG(pmaphist, "pg=%p, pp=%p, pa=%016lx, prot=%08x",
2156: pg, pp, VM_PAGE_TO_PHYS(pg), prot);
2157:
1.78 ad 2158: /* do an unlocked check first */
2159: if (atomic_load_relaxed(&pp->pp_pv.pv_pmap) == NULL &&
2160: atomic_load_relaxed(&pp->pp_pv.pv_next) == NULL) {
2161: return;
2162: }
2163:
1.63 ryo 2164: if ((prot & (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
2165: VM_PROT_NONE) {
2166: pmap_page_remove(pp, prot);
1.2 ryo 2167: } else {
1.62 ryo 2168: pmap_pv_lock(pp);
1.78 ad 2169: pv = &pp->pp_pv;
2170: while (pv != NULL) {
2171: if ((pm = pv->pv_pmap) == NULL) {
2172: KASSERT(pv == &pp->pp_pv);
2173: pv = pv->pv_next;
2174: continue;
2175: }
2176: if (!pm_reverse_lock(pm, pp)) {
2177: /* retry */
2178: pv = &pp->pp_pv;
2179: continue;
2180: }
1.62 ryo 2181: _pmap_protect_pv(pp, pv, prot);
1.78 ad 2182: pm_unlock(pm);
2183: pv = pv->pv_next;
1.2 ryo 2184: }
1.62 ryo 2185: pmap_pv_unlock(pp);
1.2 ryo 2186: }
1.1 matt 2187: }
2188:
2189: void
1.2 ryo 2190: pmap_unwire(struct pmap *pm, vaddr_t va)
1.1 matt 2191: {
1.2 ryo 2192: pt_entry_t pte, *ptep;
2193:
2194: UVMHIST_FUNC(__func__);
2195: UVMHIST_CALLED(pmaphist);
2196:
2197: UVMHIST_LOG(pmaphist, "pm=%p, va=%016lx", pm, va, 0, 0);
2198:
2199: PMAP_COUNT(unwire);
2200:
1.28 ryo 2201: KASSERT_PM_ADDR(pm, va);
2202: KASSERT(!IN_KSEG_ADDR(va));
1.2 ryo 2203:
1.11 ryo 2204: pm_lock(pm);
1.27 ryo 2205: ptep = _pmap_pte_lookup_l3(pm, va);
1.2 ryo 2206: if (ptep != NULL) {
2207: pte = *ptep;
1.3 ryo 2208: if (!l3pte_valid(pte) ||
2209: ((pte & LX_BLKPAG_OS_WIRED) == 0)) {
2210: /* invalid pte, or pte is not wired */
2211: PMAP_COUNT(unwire_failure);
1.11 ryo 2212: pm_unlock(pm);
1.2 ryo 2213: return;
2214: }
2215:
1.3 ryo 2216: pte &= ~LX_BLKPAG_OS_WIRED;
2217: atomic_swap_64(ptep, pte);
1.2 ryo 2218:
1.36 ryo 2219: PMSTAT_DEC_WIRED_COUNT(pm);
1.2 ryo 2220: }
1.11 ryo 2221: pm_unlock(pm);
1.1 matt 2222: }
2223:
1.2 ryo 2224: bool
2225: pmap_fault_fixup(struct pmap *pm, vaddr_t va, vm_prot_t accessprot, bool user)
1.1 matt 2226: {
1.62 ryo 2227: struct pmap_page *pp;
1.2 ryo 2228: pt_entry_t *ptep, pte;
2229: vm_prot_t pmap_prot;
2230: paddr_t pa;
1.8 ryo 2231: bool fixed = false;
1.2 ryo 2232:
2233: UVMHIST_FUNC(__func__);
2234: UVMHIST_CALLED(pmaphist);
2235:
2236: UVMHIST_LOG(pmaphist, "pm=%p, va=%016lx, accessprot=%08x",
2237: pm, va, accessprot, 0);
2238:
2239: #if 0
1.28 ryo 2240: KASSERT_PM_ADDR(pm, va);
1.2 ryo 2241: #else
2242: if (((pm == pmap_kernel()) &&
1.28 ryo 2243: !(IN_RANGE(va, VM_MIN_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS))) ||
1.2 ryo 2244: ((pm != pmap_kernel()) &&
1.28 ryo 2245: !(IN_RANGE(va, VM_MIN_ADDRESS, VM_MAX_ADDRESS)))) {
1.2 ryo 2246:
2247: UVMHIST_LOG(pmaphist,
1.85 ! skrll 2248: "pmap space and va mismatch: kernel=%jd, va=%016lx",
! 2249: pm == pmap_kernel(), va, 0, 0);
1.2 ryo 2250: return false;
2251: }
2252: #endif
2253:
1.11 ryo 2254: pm_lock(pm);
1.8 ryo 2255:
1.27 ryo 2256: ptep = _pmap_pte_lookup_l3(pm, va);
1.2 ryo 2257: if (ptep == NULL) {
2258: UVMHIST_LOG(pmaphist, "pte_lookup failure: va=%016lx",
2259: va, 0, 0, 0);
1.8 ryo 2260: goto done;
1.2 ryo 2261: }
2262:
2263: pte = *ptep;
2264: if (!l3pte_valid(pte)) {
2265: UVMHIST_LOG(pmaphist, "invalid pte: %016llx: va=%016lx",
2266: pte, va, 0, 0);
1.8 ryo 2267: goto done;
1.2 ryo 2268: }
2269:
2270: pa = l3pte_pa(*ptep);
1.62 ryo 2271: pp = phys_to_pp(pa);
2272: if (pp == NULL) {
2273: UVMHIST_LOG(pmaphist, "pmap_page not found: va=%016lx", va, 0, 0, 0);
1.8 ryo 2274: goto done;
1.2 ryo 2275: }
2276:
2277: /* get prot by pmap_enter() (stored in software use bit in pte) */
2278: switch (pte & (LX_BLKPAG_OS_READ|LX_BLKPAG_OS_WRITE)) {
2279: case 0:
2280: default:
2281: pmap_prot = 0;
2282: break;
2283: case LX_BLKPAG_OS_READ:
2284: pmap_prot = VM_PROT_READ;
2285: break;
2286: case LX_BLKPAG_OS_WRITE:
2287: case LX_BLKPAG_OS_READ|LX_BLKPAG_OS_WRITE:
2288: pmap_prot = (VM_PROT_READ|VM_PROT_WRITE);
2289: break;
2290: }
1.18 ryo 2291: if (l3pte_executable(pte, pm != pmap_kernel()))
1.2 ryo 2292: pmap_prot |= VM_PROT_EXECUTE;
2293:
2294: UVMHIST_LOG(pmaphist, "va=%016lx, pmapprot=%08x, accessprot=%08x",
2295: va, pmap_prot, accessprot, 0);
2296:
2297: /* ignore except read/write */
2298: accessprot &= (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE);
2299:
1.61 ryo 2300: /* PROT_EXEC requires implicit PROT_READ */
2301: if (accessprot & VM_PROT_EXECUTE)
2302: accessprot |= VM_PROT_READ;
2303:
1.2 ryo 2304: /* no permission to read/write/execute for this page */
2305: if ((pmap_prot & accessprot) != accessprot) {
2306: UVMHIST_LOG(pmaphist, "no permission to access", 0, 0, 0, 0);
1.8 ryo 2307: goto done;
1.2 ryo 2308: }
2309:
1.24 ryo 2310: /* pte is readable and writable, but occured fault? probably copy(9) */
2311: if ((pte & LX_BLKPAG_AF) && ((pte & LX_BLKPAG_AP) == LX_BLKPAG_AP_RW))
1.8 ryo 2312: goto done;
1.2 ryo 2313:
1.62 ryo 2314: pmap_pv_lock(pp);
1.2 ryo 2315: if ((pte & LX_BLKPAG_AF) == 0) {
2316: /* pte has no AF bit, set referenced and AF bit */
2317: UVMHIST_LOG(pmaphist,
2318: "REFERENCED:"
2319: " va=%016lx, pa=%016lx, pte_prot=%08x, accessprot=%08x",
2320: va, pa, pmap_prot, accessprot);
1.78 ad 2321: pp->pp_pv.pv_va |= VM_PROT_READ; /* set referenced */
1.2 ryo 2322: pte |= LX_BLKPAG_AF;
2323:
2324: PMAP_COUNT(fixup_referenced);
2325: }
2326: if ((accessprot & VM_PROT_WRITE) &&
2327: ((pte & LX_BLKPAG_AP) == LX_BLKPAG_AP_RO)) {
2328: /* pte is not RW. set modified and RW */
2329:
2330: UVMHIST_LOG(pmaphist, "MODIFIED:"
2331: " va=%016lx, pa=%016lx, pte_prot=%08x, accessprot=%08x",
2332: va, pa, pmap_prot, accessprot);
1.78 ad 2333: pp->pp_pv.pv_va |= VM_PROT_WRITE; /* set modified */
1.2 ryo 2334: pte &= ~LX_BLKPAG_AP;
2335: pte |= LX_BLKPAG_AP_RW;
2336:
2337: PMAP_COUNT(fixup_modified);
2338: }
1.62 ryo 2339: pmap_pv_unlock(pp);
1.2 ryo 2340:
2341: atomic_swap_64(ptep, pte);
1.19 ryo 2342: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
2343:
1.8 ryo 2344: fixed = true;
1.2 ryo 2345:
1.8 ryo 2346: done:
1.11 ryo 2347: pm_unlock(pm);
1.8 ryo 2348: return fixed;
1.1 matt 2349: }
2350:
2351: bool
2352: pmap_clear_modify(struct vm_page *pg)
2353: {
1.2 ryo 2354: struct pv_entry *pv;
1.62 ryo 2355: struct pmap_page * const pp = VM_PAGE_TO_PP(pg);
1.2 ryo 2356: pt_entry_t *ptep, pte, opte;
2357: vaddr_t va;
2358:
2359: UVMHIST_FUNC(__func__);
2360: UVMHIST_CALLED(pmaphist);
2361:
1.78 ad 2362: UVMHIST_LOG(pmaphist, "pg=%p, flags=%08x",
2363: pg, (int)(pp->pp_pv.pv_va & (PAGE_SIZE - 1)), 0, 0);
2364:
2365: PMAP_COUNT(clear_modify);
2366:
2367: /*
2368: * if this is a new page, assert it has no mappings and simply zap
2369: * the stored attributes without taking any locks.
2370: */
2371: if ((pg->flags & PG_FAKE) != 0) {
2372: KASSERT(atomic_load_relaxed(&pp->pp_pv.pv_pmap) == NULL);
2373: KASSERT(atomic_load_relaxed(&pp->pp_pv.pv_next) == NULL);
2374: atomic_store_relaxed(&pp->pp_pv.pv_va, 0);
2375: return false;
2376: }
1.2 ryo 2377:
1.62 ryo 2378: pmap_pv_lock(pp);
1.2 ryo 2379:
1.78 ad 2380: if ((pp->pp_pv.pv_va & VM_PROT_WRITE) == 0) {
1.62 ryo 2381: pmap_pv_unlock(pp);
1.2 ryo 2382: return false;
2383: }
2384:
1.78 ad 2385: pp->pp_pv.pv_va &= ~(vaddr_t)VM_PROT_WRITE;
2386:
2387: for (pv = &pp->pp_pv; pv != NULL; pv = pv->pv_next) {
2388: if (pv->pv_pmap == NULL) {
2389: KASSERT(pv == &pp->pp_pv);
2390: continue;
2391: }
1.2 ryo 2392:
2393: PMAP_COUNT(clear_modify_pages);
2394:
1.78 ad 2395: va = trunc_page(pv->pv_va);
1.2 ryo 2396:
2397: ptep = pv->pv_ptep;
2398: opte = pte = *ptep;
2399: tryagain:
2400: if (!l3pte_valid(pte))
2401: continue;
2402:
2403: /* clear write permission */
2404: pte &= ~LX_BLKPAG_AP;
2405: pte |= LX_BLKPAG_AP_RO;
2406:
2407: /* XXX: possible deadlock if using PM_LOCK(). this is racy */
2408: if ((pte = atomic_cas_64(ptep, opte, pte)) != opte) {
2409: opte = pte;
2410: goto tryagain;
2411: }
2412:
1.19 ryo 2413: AARCH64_TLBI_BY_ASID_VA(pv->pv_pmap->pm_asid, va, true);
1.2 ryo 2414:
2415: UVMHIST_LOG(pmaphist,
2416: "va=%016llx, ptep=%p, pa=%016lx, RW -> RO",
2417: va, ptep, l3pte_pa(pte), 0);
2418: }
2419:
1.62 ryo 2420: pmap_pv_unlock(pp);
1.2 ryo 2421:
2422: return true;
1.1 matt 2423: }
2424:
2425: bool
2426: pmap_clear_reference(struct vm_page *pg)
2427: {
1.2 ryo 2428: struct pv_entry *pv;
1.62 ryo 2429: struct pmap_page * const pp = VM_PAGE_TO_PP(pg);
1.2 ryo 2430: pt_entry_t *ptep, pte, opte;
2431: vaddr_t va;
2432:
2433: UVMHIST_FUNC(__func__);
2434: UVMHIST_CALLED(pmaphist);
2435:
1.78 ad 2436: UVMHIST_LOG(pmaphist, "pg=%p, pp=%p, flags=%08x",
2437: pg, pp, (int)(pp->pp_pv.pv_va & (PAGE_SIZE - 1)), 0);
1.2 ryo 2438:
1.62 ryo 2439: pmap_pv_lock(pp);
1.2 ryo 2440:
1.78 ad 2441: if ((pp->pp_pv.pv_va & VM_PROT_READ) == 0) {
1.62 ryo 2442: pmap_pv_unlock(pp);
1.2 ryo 2443: return false;
2444: }
1.78 ad 2445: pp->pp_pv.pv_va &= ~(vaddr_t)VM_PROT_READ;
1.2 ryo 2446:
2447: PMAP_COUNT(clear_reference);
1.78 ad 2448: for (pv = &pp->pp_pv; pv != NULL; pv = pv->pv_next) {
2449: if (pv->pv_pmap == NULL) {
2450: KASSERT(pv == &pp->pp_pv);
2451: continue;
2452: }
2453:
1.2 ryo 2454: PMAP_COUNT(clear_reference_pages);
2455:
1.78 ad 2456: va = trunc_page(pv->pv_va);
1.2 ryo 2457:
2458: ptep = pv->pv_ptep;
2459: opte = pte = *ptep;
2460: tryagain:
2461: if (!l3pte_valid(pte))
2462: continue;
2463:
2464: /* clear access permission */
2465: pte &= ~LX_BLKPAG_AF;
2466:
2467: /* XXX: possible deadlock if using PM_LOCK(). this is racy */
2468: if ((pte = atomic_cas_64(ptep, opte, pte)) != opte) {
2469: opte = pte;
2470: goto tryagain;
2471: }
2472:
1.19 ryo 2473: AARCH64_TLBI_BY_ASID_VA(pv->pv_pmap->pm_asid, va, true);
1.2 ryo 2474:
2475: UVMHIST_LOG(pmaphist, "va=%016llx, ptep=%p, pa=%016lx, unse AF",
2476: va, ptep, l3pte_pa(pte), 0);
2477: }
2478:
1.62 ryo 2479: pmap_pv_unlock(pp);
1.2 ryo 2480:
2481: return true;
1.1 matt 2482: }
2483:
2484: bool
2485: pmap_is_modified(struct vm_page *pg)
2486: {
1.62 ryo 2487: struct pmap_page * const pp = VM_PAGE_TO_PP(pg);
1.3 ryo 2488:
1.78 ad 2489: return (pp->pp_pv.pv_va & VM_PROT_WRITE);
1.1 matt 2490: }
2491:
2492: bool
2493: pmap_is_referenced(struct vm_page *pg)
2494: {
1.62 ryo 2495: struct pmap_page * const pp = VM_PAGE_TO_PP(pg);
1.3 ryo 2496:
1.78 ad 2497: return (pp->pp_pv.pv_va & VM_PROT_READ);
1.2 ryo 2498: }
2499:
2500: #ifdef DDB
1.37 ryo 2501:
1.17 ryo 2502: /* get pointer to kernel segment L2 or L3 table entry */
2503: pt_entry_t *
2504: kvtopte(vaddr_t va)
2505: {
1.28 ryo 2506: KASSERT(IN_RANGE(va, VM_MIN_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS));
1.17 ryo 2507:
1.27 ryo 2508: return _pmap_pte_lookup_bs(pmap_kernel(), va, NULL);
1.17 ryo 2509: }
2510:
2511: /* change attribute of kernel segment */
2512: pt_entry_t
2513: pmap_kvattr(vaddr_t va, vm_prot_t prot)
2514: {
2515: pt_entry_t *ptep, pte, opte;
2516:
1.28 ryo 2517: KASSERT(IN_RANGE(va, VM_MIN_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS));
1.17 ryo 2518:
2519: ptep = kvtopte(va);
2520: if (ptep == NULL)
2521: panic("%s: %016lx is not mapped\n", __func__, va);
2522:
2523: opte = pte = *ptep;
2524:
2525: pte &= ~(LX_BLKPAG_AF|LX_BLKPAG_AP);
2526: switch (prot & (VM_PROT_READ|VM_PROT_WRITE)) {
2527: case 0:
2528: break;
2529: case VM_PROT_READ:
2530: pte |= (LX_BLKPAG_AF|LX_BLKPAG_AP_RO);
2531: break;
2532: case VM_PROT_WRITE:
2533: case VM_PROT_READ|VM_PROT_WRITE:
2534: pte |= (LX_BLKPAG_AF|LX_BLKPAG_AP_RW);
2535: break;
2536: }
2537:
2538: if ((prot & VM_PROT_EXECUTE) == 0) {
1.18 ryo 2539: pte |= LX_BLKPAG_PXN;
1.17 ryo 2540: } else {
2541: pte |= LX_BLKPAG_AF;
1.18 ryo 2542: pte &= ~LX_BLKPAG_PXN;
1.17 ryo 2543: }
2544:
2545: *ptep = pte;
2546:
2547: return opte;
2548: }
2549:
1.26 ryo 2550: void
2551: pmap_db_pte_print(pt_entry_t pte, int level,
2552: void (*pr)(const char *, ...) __printflike(1, 2))
1.2 ryo 2553: {
2554: if (pte == 0) {
1.8 ryo 2555: pr(" UNUSED\n");
1.26 ryo 2556: return;
2557: }
2558:
2559: pr(" %s", (pte & LX_VALID) ? "VALID" : "**INVALID**");
2560:
2561: if ((level == 0) ||
2562: ((level == 1) && l1pde_is_table(pte)) ||
2563: ((level == 2) && l2pde_is_table(pte))) {
2564:
2565: /* L0/L1/L2 TABLE */
2566: if ((level == 0) && ((pte & LX_TYPE) != LX_TYPE_TBL))
2567: pr(" **ILLEGAL TYPE**"); /* L0 doesn't support block */
2568: else
1.37 ryo 2569: pr(" L%d-TABLE", level);
1.2 ryo 2570:
1.26 ryo 2571: pr(", PA=%lx", l0pde_pa(pte));
1.2 ryo 2572:
1.26 ryo 2573: if (pte & LX_TBL_NSTABLE)
2574: pr(", NSTABLE");
2575: if (pte & LX_TBL_APTABLE)
2576: pr(", APTABLE");
2577: if (pte & LX_TBL_UXNTABLE)
2578: pr(", UXNTABLE");
2579: if (pte & LX_TBL_PXNTABLE)
2580: pr(", PXNTABLE");
1.2 ryo 2581:
2582: } else if (((level == 1) && l1pde_is_block(pte)) ||
2583: ((level == 2) && l2pde_is_block(pte)) ||
1.26 ryo 2584: (level == 3)) {
1.2 ryo 2585:
1.26 ryo 2586: /* L1/L2 BLOCK or L3 PAGE */
1.37 ryo 2587: switch (level) {
2588: case 1:
2589: pr(" L1(1G)-BLOCK");
2590: break;
2591: case 2:
2592: pr(" L2(2M)-BLOCK");
2593: break;
2594: case 3:
1.26 ryo 2595: pr(" %s", l3pte_is_page(pte) ?
1.37 ryo 2596: "L3(4K)-PAGE" : "**ILLEGAL TYPE**");
2597: break;
2598: }
1.2 ryo 2599:
1.26 ryo 2600: pr(", PA=%lx", l3pte_pa(pte));
1.2 ryo 2601:
1.37 ryo 2602: pr(", %s", (pte & LX_BLKPAG_UXN) ?
1.43 skrll 2603: "UXN" : "UX ");
1.37 ryo 2604: pr(", %s", (pte & LX_BLKPAG_PXN) ?
1.43 skrll 2605: "PXN" : "PX ");
1.2 ryo 2606:
2607: if (pte & LX_BLKPAG_CONTIG)
1.26 ryo 2608: pr(", CONTIG");
1.2 ryo 2609:
1.26 ryo 2610: pr(", %s", (pte & LX_BLKPAG_NG) ? "NG" : "global");
1.37 ryo 2611: pr(", %s", (pte & LX_BLKPAG_AF) ?
2612: "accessible" :
2613: "**fault** ");
1.2 ryo 2614:
2615: switch (pte & LX_BLKPAG_SH) {
2616: case LX_BLKPAG_SH_NS:
2617: pr(", SH_NS");
2618: break;
2619: case LX_BLKPAG_SH_OS:
2620: pr(", SH_OS");
2621: break;
2622: case LX_BLKPAG_SH_IS:
2623: pr(", SH_IS");
2624: break;
2625: default:
2626: pr(", SH_??");
2627: break;
2628: }
2629:
2630: pr(", %s", (pte & LX_BLKPAG_AP_RO) ? "RO" : "RW");
2631: pr(", %s", (pte & LX_BLKPAG_APUSER) ? "EL0" : "EL1");
1.26 ryo 2632: pr(", %s", (pte & LX_BLKPAG_NS) ? "NS" : "secure");
1.2 ryo 2633:
2634: switch (pte & LX_BLKPAG_ATTR_MASK) {
2635: case LX_BLKPAG_ATTR_NORMAL_WB:
1.43 skrll 2636: pr(", WB");
1.2 ryo 2637: break;
2638: case LX_BLKPAG_ATTR_NORMAL_NC:
1.43 skrll 2639: pr(", NC");
1.2 ryo 2640: break;
2641: case LX_BLKPAG_ATTR_NORMAL_WT:
1.43 skrll 2642: pr(", WT");
1.2 ryo 2643: break;
2644: case LX_BLKPAG_ATTR_DEVICE_MEM:
2645: pr(", DEVICE");
2646: break;
2647: }
2648:
1.26 ryo 2649: if (pte & LX_BLKPAG_OS_BOOT)
2650: pr(", boot");
1.2 ryo 2651: if (pte & LX_BLKPAG_OS_READ)
2652: pr(", pmap_read");
2653: if (pte & LX_BLKPAG_OS_WRITE)
2654: pr(", pmap_write");
1.26 ryo 2655: if (pte & LX_BLKPAG_OS_WIRED)
1.43 skrll 2656: pr(", wired");
1.2 ryo 2657: } else {
1.26 ryo 2658: pr(" **ILLEGAL TYPE**");
1.2 ryo 2659: }
2660: pr("\n");
1.1 matt 2661: }
2662:
1.2 ryo 2663: void
1.38 ryo 2664: pmap_db_pteinfo(vaddr_t va, void (*pr)(const char *, ...) __printflike(1, 2))
1.1 matt 2665: {
1.2 ryo 2666: struct vm_page *pg;
1.62 ryo 2667: struct pmap_page *pp;
1.2 ryo 2668: bool user;
2669: pd_entry_t *l0, *l1, *l2, *l3;
2670: pd_entry_t pde;
2671: pt_entry_t pte;
1.26 ryo 2672: uint64_t ttbr;
1.2 ryo 2673: paddr_t pa;
2674: unsigned int idx;
2675:
1.66 ryo 2676: switch (aarch64_addressspace(va)) {
2677: case AARCH64_ADDRSPACE_UPPER:
1.26 ryo 2678: user = false;
2679: ttbr = reg_ttbr1_el1_read();
1.66 ryo 2680: break;
2681: case AARCH64_ADDRSPACE_LOWER:
1.26 ryo 2682: user = true;
2683: ttbr = reg_ttbr0_el1_read();
1.66 ryo 2684: break;
2685: default:
2686: pr("illegal address space\n");
2687: return;
1.26 ryo 2688: }
2689: pa = ttbr & TTBR_BADDR;
2690: l0 = (pd_entry_t *)AARCH64_PA_TO_KVA(pa);
2691:
1.2 ryo 2692: /*
2693: * traverse L0 -> L1 -> L2 -> L3 table
2694: */
1.38 ryo 2695: pr("TTBR%d=%016"PRIx64", pa=%016"PRIxPADDR", va=%p",
2696: user ? 0 : 1, ttbr, pa, l0);
2697: pr(", input-va=%016"PRIxVADDR
2698: ", L0-index=%ld, L1-index=%ld, L2-index=%ld, L3-index=%ld\n",
1.2 ryo 2699: va,
2700: (va & L0_ADDR_BITS) >> L0_SHIFT,
2701: (va & L1_ADDR_BITS) >> L1_SHIFT,
2702: (va & L2_ADDR_BITS) >> L2_SHIFT,
2703: (va & L3_ADDR_BITS) >> L3_SHIFT);
2704:
2705: idx = l0pde_index(va);
2706: pde = l0[idx];
2707:
1.38 ryo 2708: pr("L0[%3d]=%016"PRIx64":", idx, pde);
1.2 ryo 2709: pmap_db_pte_print(pde, 0, pr);
2710:
2711: if (!l0pde_valid(pde))
2712: return;
2713:
1.26 ryo 2714: l1 = (pd_entry_t *)AARCH64_PA_TO_KVA(l0pde_pa(pde));
1.2 ryo 2715: idx = l1pde_index(va);
2716: pde = l1[idx];
2717:
1.38 ryo 2718: pr(" L1[%3d]=%016"PRIx64":", idx, pde);
1.2 ryo 2719: pmap_db_pte_print(pde, 1, pr);
2720:
2721: if (!l1pde_valid(pde) || l1pde_is_block(pde))
2722: return;
2723:
1.26 ryo 2724: l2 = (pd_entry_t *)AARCH64_PA_TO_KVA(l1pde_pa(pde));
1.2 ryo 2725: idx = l2pde_index(va);
2726: pde = l2[idx];
2727:
1.38 ryo 2728: pr(" L2[%3d]=%016"PRIx64":", idx, pde);
1.2 ryo 2729: pmap_db_pte_print(pde, 2, pr);
2730:
2731: if (!l2pde_valid(pde) || l2pde_is_block(pde))
2732: return;
2733:
1.26 ryo 2734: l3 = (pd_entry_t *)AARCH64_PA_TO_KVA(l2pde_pa(pde));
1.2 ryo 2735: idx = l3pte_index(va);
2736: pte = l3[idx];
2737:
1.38 ryo 2738: pr(" L3[%3d]=%016"PRIx64":", idx, pte);
1.2 ryo 2739: pmap_db_pte_print(pte, 3, pr);
2740:
2741: pa = l3pte_pa(pte);
2742: pg = PHYS_TO_VM_PAGE(pa);
1.62 ryo 2743: pp = phys_to_pp(pa);
2744: if (pp == NULL) {
2745: pr("No VM_PAGE nor PMAP_PAGE\n");
1.8 ryo 2746: } else {
1.62 ryo 2747: if (pg != NULL)
2748: pg_dump(pg, pr);
2749: else
2750: pr("no VM_PAGE. pv-tracked page?\n");
2751: pv_dump(pp, pr);
1.2 ryo 2752: }
1.1 matt 2753: }
1.37 ryo 2754:
2755: static void
2756: dump_ln_table(bool countmode, pd_entry_t *pdp, int level, int lnindex,
1.38 ryo 2757: vaddr_t va, void (*pr)(const char *, ...) __printflike(1, 2))
1.37 ryo 2758: {
2759: struct vm_page *pg;
2760: struct vm_page_md *md;
2761: pd_entry_t pde;
2762: paddr_t pa;
2763: int i, n;
2764: const char *spaces[4] = { " ", " ", " ", " " };
2765: const char *spc = spaces[level];
2766:
2767: pa = AARCH64_KVA_TO_PA((vaddr_t)pdp);
2768: pg = PHYS_TO_VM_PAGE(pa);
2769: md = VM_PAGE_TO_MD(pg);
2770:
2771: if (pg == NULL) {
2772: pr("%sL%d: pa=%lx pg=NULL\n", spc, level, pa);
2773: } else {
2774: pr("%sL%d: pa=%lx pg=%p, wire_count=%d, mdpg_ptep_parent=%p\n",
2775: spc, level, pa, pg, pg->wire_count, md->mdpg_ptep_parent);
2776: }
2777:
2778: for (i = n = 0; i < Ln_ENTRIES; i++) {
2779: db_read_bytes((db_addr_t)&pdp[i], sizeof(pdp[i]), (char *)&pde);
2780: if (lxpde_valid(pde)) {
2781: if (!countmode)
2782: pr("%sL%d[%3d] %3dth, va=%016lx, pte=%016lx:",
2783: spc, level, i, n, va, pde);
2784: n++;
2785:
2786: if (((level != 0) && (level != 3) &&
2787: l1pde_is_block(pde)) ||
2788: ((level == 3) && l3pte_is_page(pde))) {
2789: if (!countmode)
2790: pmap_db_pte_print(pde, level, pr);
2791: } else if ((level != 3) && l1pde_is_table(pde)) {
2792: if (!countmode)
2793: pmap_db_pte_print(pde, level, pr);
2794: pa = l0pde_pa(pde);
2795: dump_ln_table(countmode,
2796: (pd_entry_t *)AARCH64_PA_TO_KVA(pa),
2797: level + 1, i, va, pr);
2798: } else {
2799: if (!countmode)
2800: pmap_db_pte_print(pde, level, pr);
2801: }
2802: }
2803:
2804: switch (level) {
2805: case 0:
2806: va += L0_SIZE;
2807: break;
2808: case 1:
2809: va += L1_SIZE;
2810: break;
2811: case 2:
2812: va += L2_SIZE;
2813: break;
2814: case 3:
2815: va += L3_SIZE;
2816: break;
2817: }
2818: }
2819:
2820: if (level == 0)
2821: pr("L0 has %d entries\n", n);
2822: else
2823: pr("%sL%d[%3d] has %d L%d entries\n", spaces[level - 1],
2824: level - 1, lnindex, n, level);
2825:
2826: }
2827:
2828: static void
2829: pmap_db_dump_l0_table(bool countmode, pd_entry_t *pdp, vaddr_t va_base,
1.38 ryo 2830: void (*pr)(const char *, ...) __printflike(1, 2))
1.37 ryo 2831: {
2832: dump_ln_table(countmode, pdp, 0, 0, va_base, pr);
2833: }
2834:
2835: void
1.38 ryo 2836: pmap_db_ttbrdump(bool countmode, vaddr_t va,
2837: void (*pr)(const char *, ...) __printflike(1, 2))
1.37 ryo 2838: {
2839: struct pmap *pm, _pm;
2840:
2841: pm = (struct pmap *)va;
2842: db_read_bytes((db_addr_t)va, sizeof(_pm), (char *)&_pm);
2843:
2844: pr("pmap=%p\n", pm);
2845: pr(" pm_asid = %d\n", _pm.pm_asid);
2846: pr(" pm_l0table = %p\n", _pm.pm_l0table);
2847: pr(" pm_l0table_pa = %lx\n", _pm.pm_l0table_pa);
2848: pr(" pm_activated = %d\n\n", _pm.pm_activated);
2849:
2850: pmap_db_dump_l0_table(countmode, _pm.pm_l0table,
2851: (pm == pmap_kernel()) ? 0xffff000000000000UL : 0, pr);
2852: }
2853:
1.2 ryo 2854: #endif /* DDB */
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to pmap.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / aarch64 / aarch64