Annotation of src/sys/arch/aarch64/aarch64/pmap.c, Revision 1.61
1.61 ! ryo 1: /* $NetBSD: pmap.c,v 1.60 2019/12/30 16:03:48 skrll Exp $ */
1.1 matt 2:
1.2 ryo 3: /*
4: * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
1.1 matt 5: * All rights reserved.
6: *
7: * Redistribution and use in source and binary forms, with or without
8: * modification, are permitted provided that the following conditions
9: * are met:
10: * 1. Redistributions of source code must retain the above copyright
11: * notice, this list of conditions and the following disclaimer.
12: * 2. Redistributions in binary form must reproduce the above copyright
13: * notice, this list of conditions and the following disclaimer in the
14: * documentation and/or other materials provided with the distribution.
15: *
1.2 ryo 16: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18: * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19: * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
20: * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21: * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22: * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
24: * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
25: * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
1.1 matt 26: * POSSIBILITY OF SUCH DAMAGE.
27: */
28:
29: #include <sys/cdefs.h>
1.61 ! ryo 30: __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.60 2019/12/30 16:03:48 skrll Exp $");
1.1 matt 31:
1.2 ryo 32: #include "opt_arm_debug.h"
33: #include "opt_ddb.h"
1.26 ryo 34: #include "opt_multiprocessor.h"
35: #include "opt_pmap.h"
1.2 ryo 36: #include "opt_uvmhist.h"
1.1 matt 37:
38: #include <sys/param.h>
39: #include <sys/types.h>
40: #include <sys/kmem.h>
41: #include <sys/vmem.h>
1.2 ryo 42: #include <sys/atomic.h>
1.33 maxv 43: #include <sys/asan.h>
1.1 matt 44:
45: #include <uvm/uvm.h>
46:
1.2 ryo 47: #include <aarch64/pmap.h>
48: #include <aarch64/pte.h>
49: #include <aarch64/armreg.h>
50: #include <aarch64/cpufunc.h>
1.21 ryo 51: #include <aarch64/machdep.h>
1.37 ryo 52: #ifdef DDB
53: #include <aarch64/db_machdep.h>
54: #include <ddb/db_access.h>
55: #endif
1.2 ryo 56:
57: //#define PMAP_DEBUG
58: //#define PMAP_PV_DEBUG
59:
1.16 skrll 60: #ifdef VERBOSE_INIT_ARM
61: #define VPRINTF(...) printf(__VA_ARGS__)
62: #else
1.30 skrll 63: #define VPRINTF(...) __nothing
1.16 skrll 64: #endif
1.2 ryo 65:
66: UVMHIST_DEFINE(pmaphist);
67: #ifdef UVMHIST
68:
69: #ifndef UVMHIST_PMAPHIST_SIZE
70: #define UVMHIST_PMAPHIST_SIZE (1024 * 4)
71: #endif
72:
73: struct kern_history_ent pmaphistbuf[UVMHIST_PMAPHIST_SIZE];
74:
75: static void
76: pmap_hist_init(void)
77: {
78: static bool inited = false;
79: if (inited == false) {
80: UVMHIST_INIT_STATIC(pmaphist, pmaphistbuf);
81: inited = true;
82: }
83: }
84: #define PMAP_HIST_INIT() pmap_hist_init()
85:
86: #else /* UVMHIST */
87:
88: #define PMAP_HIST_INIT() ((void)0)
89:
90: #endif /* UVMHIST */
91:
92:
93: #ifdef PMAPCOUNTERS
94: #define PMAP_COUNT(name) (pmap_evcnt_##name.ev_count++ + 0)
95: #define PMAP_COUNTER(name, desc) \
96: struct evcnt pmap_evcnt_##name = \
97: EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", desc); \
98: EVCNT_ATTACH_STATIC(pmap_evcnt_##name)
99:
100: PMAP_COUNTER(pdp_alloc_boot, "page table page allocate (uvm_pageboot_alloc)");
101: PMAP_COUNTER(pdp_alloc, "page table page allocate (uvm_pagealloc)");
102: PMAP_COUNTER(pdp_free, "page table page free (uvm_pagefree)");
103:
104: PMAP_COUNTER(pv_enter, "pv_entry allocate and link");
105: PMAP_COUNTER(pv_remove, "pv_entry free and unlink");
1.5 ryo 106: PMAP_COUNTER(pv_remove_nopv, "no pv_entry found when removing pv");
1.2 ryo 107:
108: PMAP_COUNTER(activate, "pmap_activate call");
109: PMAP_COUNTER(deactivate, "pmap_deactivate call");
110: PMAP_COUNTER(create, "pmap_create call");
111: PMAP_COUNTER(destroy, "pmap_destroy call");
112:
113: PMAP_COUNTER(page_protect, "pmap_page_protect call");
114: PMAP_COUNTER(protect, "pmap_protect call");
115: PMAP_COUNTER(protect_remove_fallback, "pmap_protect with no-read");
116: PMAP_COUNTER(protect_none, "pmap_protect non-exists pages");
117: PMAP_COUNTER(protect_managed, "pmap_protect managed pages");
118: PMAP_COUNTER(protect_unmanaged, "pmap_protect unmanaged pages");
119:
120: PMAP_COUNTER(clear_modify, "pmap_clear_modify call");
121: PMAP_COUNTER(clear_modify_pages, "pmap_clear_modify pages");
122: PMAP_COUNTER(clear_reference, "pmap_clear_reference call");
123: PMAP_COUNTER(clear_reference_pages, "pmap_clear_reference pages");
124:
125: PMAP_COUNTER(fixup_referenced, "page reference emulations");
126: PMAP_COUNTER(fixup_modified, "page modification emulations");
127:
128: PMAP_COUNTER(kern_mappings_bad, "kernel pages mapped (bad color)");
129: PMAP_COUNTER(kern_mappings_bad_wired, "kernel pages mapped (wired bad color)");
130: PMAP_COUNTER(user_mappings_bad, "user pages mapped (bad color, not wired)");
131: PMAP_COUNTER(user_mappings_bad_wired, "user pages mapped (bad colo, wiredr)");
132: PMAP_COUNTER(kern_mappings, "kernel pages mapped");
133: PMAP_COUNTER(user_mappings, "user pages mapped");
134: PMAP_COUNTER(user_mappings_changed, "user mapping changed");
135: PMAP_COUNTER(kern_mappings_changed, "kernel mapping changed");
136: PMAP_COUNTER(uncached_mappings, "uncached pages mapped");
137: PMAP_COUNTER(unmanaged_mappings, "unmanaged pages mapped");
138: PMAP_COUNTER(managed_mappings, "managed pages mapped");
139: PMAP_COUNTER(mappings, "pages mapped (including remapped)");
140: PMAP_COUNTER(remappings, "pages remapped");
141:
142: PMAP_COUNTER(pv_entry_cannotalloc, "pv_entry allocation failure");
143:
144: PMAP_COUNTER(unwire, "pmap_unwire call");
1.3 ryo 145: PMAP_COUNTER(unwire_failure, "pmap_unwire failure");
1.2 ryo 146:
147: #else /* PMAPCOUNTERS */
148: #define PMAP_COUNT(name) __nothing
149: #endif /* PMAPCOUNTERS */
150:
1.19 ryo 151: /*
152: * invalidate TLB entry for ASID and VA.
153: * `ll' invalidates only the Last Level (usually L3) of TLB entry
154: */
155: #define AARCH64_TLBI_BY_ASID_VA(asid, va, ll) \
156: do { \
157: if ((ll)) { \
158: if ((asid) == 0) \
159: aarch64_tlbi_by_va_ll((va)); \
160: else \
161: aarch64_tlbi_by_asid_va_ll((asid), (va)); \
162: } else { \
163: if ((asid) == 0) \
164: aarch64_tlbi_by_va((va)); \
165: else \
166: aarch64_tlbi_by_asid_va((asid), (va)); \
167: } \
168: } while (0/*CONSTCOND*/)
169:
170: /*
171: * aarch64 require write permission in pte to invalidate instruction cache.
172: * changing pte to writable temporarly before cpu_icache_sync_range().
173: * this macro modifies PTE (*ptep). need to update PTE after this.
174: */
175: #define PTE_ICACHE_SYNC_PAGE(pte, ptep, pm, va, ll) \
176: do { \
177: pt_entry_t tpte; \
178: tpte = (pte) & ~(LX_BLKPAG_AF|LX_BLKPAG_AP); \
179: tpte |= (LX_BLKPAG_AF|LX_BLKPAG_AP_RW); \
180: tpte |= (LX_BLKPAG_UXN|LX_BLKPAG_PXN); \
181: atomic_swap_64((ptep), tpte); \
182: AARCH64_TLBI_BY_ASID_VA((pm)->pm_asid, (va), (ll)); \
183: cpu_icache_sync_range((va), PAGE_SIZE); \
184: } while (0/*CONSTCOND*/)
185:
1.2 ryo 186: struct pv_entry {
187: TAILQ_ENTRY(pv_entry) pv_link;
188: struct pmap *pv_pmap;
189: vaddr_t pv_va;
190: paddr_t pv_pa; /* debug */
191: pt_entry_t *pv_ptep; /* for fast pte lookup */
192: };
1.35 ryo 193: #define pv_next pv_link.tqe_next
194:
195: #define L3INDEXMASK (L3_SIZE * Ln_ENTRIES - 1)
1.40 ryo 196: #define PDPSWEEP_TRIGGER 512
1.2 ryo 197:
1.27 ryo 198: static pt_entry_t *_pmap_pte_lookup_l3(struct pmap *, vaddr_t);
199: static pt_entry_t *_pmap_pte_lookup_bs(struct pmap *, vaddr_t, vsize_t *);
1.18 ryo 200: static pt_entry_t _pmap_pte_adjust_prot(pt_entry_t, vm_prot_t, vm_prot_t, bool);
1.2 ryo 201: static pt_entry_t _pmap_pte_adjust_cacheflags(pt_entry_t, u_int);
1.36 ryo 202: static void _pmap_remove(struct pmap *, vaddr_t, vaddr_t, bool,
203: struct pv_entry **);
1.2 ryo 204: static int _pmap_enter(struct pmap *, vaddr_t, paddr_t, vm_prot_t, u_int, bool);
205:
1.1 matt 206: static struct pmap kernel_pmap;
207:
208: struct pmap * const kernel_pmap_ptr = &kernel_pmap;
1.2 ryo 209: static vaddr_t pmap_maxkvaddr;
210:
211: vaddr_t virtual_avail, virtual_end;
212: vaddr_t virtual_devmap_addr;
1.45 ryo 213: bool pmap_devmap_bootstrap_done = false;
1.2 ryo 214:
215: static struct pool_cache _pmap_cache;
216: static struct pool_cache _pmap_pv_pool;
217:
218:
219: static inline void
220: pmap_pv_lock(struct vm_page_md *md)
221: {
222:
223: mutex_enter(&md->mdpg_pvlock);
224: }
225:
226: static inline void
227: pmap_pv_unlock(struct vm_page_md *md)
228: {
229:
230: mutex_exit(&md->mdpg_pvlock);
231: }
232:
1.11 ryo 233:
234: static inline void
235: pm_lock(struct pmap *pm)
236: {
237: mutex_enter(&pm->pm_lock);
238: }
239:
240: static inline void
241: pm_unlock(struct pmap *pm)
242: {
243: mutex_exit(&pm->pm_lock);
244: }
1.2 ryo 245:
1.28 ryo 246: #define IN_RANGE(va,sta,end) (((sta) <= (va)) && ((va) < (end)))
247:
1.2 ryo 248: #define IN_KSEG_ADDR(va) \
1.28 ryo 249: IN_RANGE((va), AARCH64_KSEG_START, AARCH64_KSEG_END)
250:
251: #define KASSERT_PM_ADDR(pm, va) \
252: do { \
253: if ((pm) == pmap_kernel()) { \
254: KASSERTMSG(IN_RANGE((va), VM_MIN_KERNEL_ADDRESS, \
255: VM_MAX_KERNEL_ADDRESS), \
256: "%s: kernel pm %p: va=%016lx" \
257: " is not kernel address\n", \
258: __func__, (pm), (va)); \
259: } else { \
260: KASSERTMSG(IN_RANGE((va), \
261: VM_MIN_ADDRESS, VM_MAX_ADDRESS), \
262: "%s: user pm %p: va=%016lx" \
263: " is not user address\n", \
264: __func__, (pm), (va)); \
265: } \
266: } while (0 /* CONSTCOND */)
1.2 ryo 267:
268:
269: static const struct pmap_devmap *pmap_devmap_table;
270:
1.60 skrll 271: #define L1_BLK_MAPPABLE_P(va, pa, size) \
272: ((((va) | (pa)) & L1_OFFSET) == 0 && (size) >= L1_SIZE)
273:
274: #define L2_BLK_MAPPABLE_P(va, pa, size) \
275: ((((va) | (pa)) & L2_OFFSET) == 0 && (size) >= L2_SIZE)
276:
1.2 ryo 277: static vsize_t
1.26 ryo 278: pmap_map_chunk(vaddr_t va, paddr_t pa, vsize_t size,
1.2 ryo 279: vm_prot_t prot, u_int flags)
280: {
281: pt_entry_t attr;
1.26 ryo 282: psize_t blocksize;
283: int rc;
1.2 ryo 284:
1.60 skrll 285: vsize_t resid = round_page(size);
286: vsize_t mapped = 0;
287:
288: while (resid > 0) {
289: if (L1_BLK_MAPPABLE_P(va, pa, resid)) {
290: blocksize = L1_SIZE;
291: attr = L1_BLOCK;
292: } else if (L2_BLK_MAPPABLE_P(va, pa, resid)) {
293: blocksize = L2_SIZE;
294: attr = L2_BLOCK;
295: } else {
296: blocksize = L3_SIZE;
297: attr = L3_PAGE;
298: }
299:
300: attr = _pmap_pte_adjust_prot(attr, prot, VM_PROT_ALL, false);
301: attr = _pmap_pte_adjust_cacheflags(attr, flags);
1.2 ryo 302:
1.60 skrll 303: rc = pmapboot_enter(va, pa, blocksize, blocksize, attr,
304: PMAPBOOT_ENTER_NOOVERWRITE, bootpage_alloc, NULL);
305: if (rc != 0) {
306: panic("%s: pmapboot_enter failed. %lx is already mapped?\n",
307: __func__, va);
308: }
1.2 ryo 309:
1.60 skrll 310: va += blocksize;
311: pa += blocksize;
312: resid -= blocksize;
313: mapped += blocksize;
1.2 ryo 314:
1.60 skrll 315: aarch64_tlbi_by_va(va);
316: }
1.2 ryo 317:
1.60 skrll 318: return mapped;
1.2 ryo 319: }
1.1 matt 320:
321: void
1.2 ryo 322: pmap_devmap_register(const struct pmap_devmap *table)
1.1 matt 323: {
1.2 ryo 324: pmap_devmap_table = table;
1.1 matt 325: }
326:
327: void
1.31 skrll 328: pmap_devmap_bootstrap(vaddr_t l0pt, const struct pmap_devmap *table)
1.2 ryo 329: {
330: vaddr_t va;
331: int i;
332:
333: pmap_devmap_register(table);
334:
1.16 skrll 335: VPRINTF("%s:\n", __func__);
1.2 ryo 336: for (i = 0; table[i].pd_size != 0; i++) {
1.16 skrll 337: VPRINTF(" devmap: pa %08lx-%08lx = va %016lx\n",
1.2 ryo 338: table[i].pd_pa,
339: table[i].pd_pa + table[i].pd_size - 1,
340: table[i].pd_va);
341: va = table[i].pd_va;
342:
1.26 ryo 343: KASSERT((VM_KERNEL_IO_ADDRESS <= va) &&
344: (va < (VM_KERNEL_IO_ADDRESS + VM_KERNEL_IO_SIZE)));
345:
1.2 ryo 346: /* update and check virtual_devmap_addr */
1.59 skrll 347: if (virtual_devmap_addr == 0 || virtual_devmap_addr > va) {
1.2 ryo 348: virtual_devmap_addr = va;
349: }
350:
1.26 ryo 351: pmap_map_chunk(
1.2 ryo 352: table[i].pd_va,
353: table[i].pd_pa,
354: table[i].pd_size,
355: table[i].pd_prot,
356: table[i].pd_flags);
357: }
1.45 ryo 358:
359: pmap_devmap_bootstrap_done = true;
1.2 ryo 360: }
361:
362: const struct pmap_devmap *
363: pmap_devmap_find_va(vaddr_t va, vsize_t size)
364: {
365: paddr_t endva;
366: int i;
367:
368: if (pmap_devmap_table == NULL)
369: return NULL;
370:
371: endva = va + size;
372: for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) {
373: if ((va >= pmap_devmap_table[i].pd_va) &&
374: (endva <= pmap_devmap_table[i].pd_va +
375: pmap_devmap_table[i].pd_size))
376: return &pmap_devmap_table[i];
377: }
378: return NULL;
379: }
380:
381: const struct pmap_devmap *
382: pmap_devmap_find_pa(paddr_t pa, psize_t size)
383: {
384: paddr_t endpa;
385: int i;
386:
387: if (pmap_devmap_table == NULL)
388: return NULL;
389:
390: endpa = pa + size;
391: for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) {
392: if (pa >= pmap_devmap_table[i].pd_pa &&
393: (endpa <= pmap_devmap_table[i].pd_pa +
394: pmap_devmap_table[i].pd_size))
395: return (&pmap_devmap_table[i]);
396: }
397: return NULL;
398: }
399:
400: vaddr_t
401: pmap_devmap_phystov(paddr_t pa)
1.1 matt 402: {
1.2 ryo 403: const struct pmap_devmap *table;
404: paddr_t offset;
405:
406: table = pmap_devmap_find_pa(pa, 0);
407: if (table == NULL)
408: return 0;
409:
410: offset = pa - table->pd_pa;
411: return table->pd_va + offset;
1.1 matt 412: }
413:
414: vaddr_t
1.2 ryo 415: pmap_devmap_vtophys(paddr_t va)
416: {
417: const struct pmap_devmap *table;
418: vaddr_t offset;
419:
420: table = pmap_devmap_find_va(va, 0);
421: if (table == NULL)
422: return 0;
423:
424: offset = va - table->pd_va;
425: return table->pd_pa + offset;
426: }
427:
428: void
429: pmap_bootstrap(vaddr_t vstart, vaddr_t vend)
430: {
431: struct pmap *kpm;
432: pd_entry_t *l0;
433: paddr_t l0pa;
434:
435: PMAP_HIST_INIT(); /* init once */
436:
437: UVMHIST_FUNC(__func__);
438: UVMHIST_CALLED(pmaphist);
439:
440: #if 0
441: /* uvmexp.ncolors = icachesize / icacheways / PAGE_SIZE; */
442: uvmexp.ncolors = aarch64_cache_vindexsize / PAGE_SIZE;
443: #endif
444:
445: /* devmap already uses last of va? */
1.59 skrll 446: if (virtual_devmap_addr != 0 && virtual_devmap_addr < vend)
1.2 ryo 447: vend = virtual_devmap_addr;
448:
449: virtual_avail = vstart;
450: virtual_end = vend;
451: pmap_maxkvaddr = vstart;
452:
453: aarch64_tlbi_all();
454:
455: l0pa = reg_ttbr1_el1_read();
1.9 christos 456: l0 = (void *)AARCH64_PA_TO_KVA(l0pa);
1.2 ryo 457:
458: memset(&kernel_pmap, 0, sizeof(kernel_pmap));
459: kpm = pmap_kernel();
460: kpm->pm_asid = 0;
461: kpm->pm_refcnt = 1;
1.40 ryo 462: kpm->pm_idlepdp = 0;
1.2 ryo 463: kpm->pm_l0table = l0;
464: kpm->pm_l0table_pa = l0pa;
465: kpm->pm_activated = true;
1.36 ryo 466: TAILQ_INIT(&kpm->pm_vmlist);
1.2 ryo 467: mutex_init(&kpm->pm_lock, MUTEX_DEFAULT, IPL_VM);
1.36 ryo 468:
469: CTASSERT(sizeof(kpm->pm_stats.wired_count) == sizeof(long));
470: CTASSERT(sizeof(kpm->pm_stats.resident_count) == sizeof(long));
471: #define PMSTAT_INC_WIRED_COUNT(pm) \
472: atomic_inc_ulong(&(pm)->pm_stats.wired_count)
473: #define PMSTAT_DEC_WIRED_COUNT(pm) \
474: atomic_dec_ulong(&(pm)->pm_stats.wired_count)
475: #define PMSTAT_INC_RESIDENT_COUNT(pm) \
476: atomic_inc_ulong(&(pm)->pm_stats.resident_count)
477: #define PMSTAT_DEC_RESIDENT_COUNT(pm) \
478: atomic_dec_ulong(&(pm)->pm_stats.resident_count)
1.2 ryo 479: }
480:
481: inline static int
482: _pmap_color(vaddr_t addr) /* or paddr_t */
483: {
484: return (addr >> PGSHIFT) & (uvmexp.ncolors - 1);
485: }
486:
487: static int
488: _pmap_pmap_ctor(void *arg, void *v, int flags)
1.1 matt 489: {
1.2 ryo 490: memset(v, 0, sizeof(struct pmap));
1.1 matt 491: return 0;
492: }
493:
1.2 ryo 494: static int
495: _pmap_pv_ctor(void *arg, void *v, int flags)
1.1 matt 496: {
1.2 ryo 497: memset(v, 0, sizeof(struct pv_entry));
498: return 0;
1.1 matt 499: }
500:
501: void
1.2 ryo 502: pmap_init(void)
1.1 matt 503: {
1.2 ryo 504: struct vm_page *pg;
505: struct vm_page_md *md;
506: uvm_physseg_t i;
507: paddr_t pfn;
508:
509: pool_cache_bootstrap(&_pmap_cache, sizeof(struct pmap),
510: 0, 0, 0, "pmappl", NULL, IPL_NONE, _pmap_pmap_ctor, NULL, NULL);
511: pool_cache_bootstrap(&_pmap_pv_pool, sizeof(struct pv_entry),
512: 0, 0, 0, "pvpl", NULL, IPL_VM, _pmap_pv_ctor, NULL, NULL);
513:
514: /*
515: * initialize vm_page_md:mdpg_pvlock at this time.
516: * When LOCKDEBUG, mutex_init() calls km_alloc,
517: * but VM_MDPAGE_INIT() is called before initialized kmem_vm_arena.
518: */
519: for (i = uvm_physseg_get_first();
520: uvm_physseg_valid_p(i);
521: i = uvm_physseg_get_next(i)) {
522: for (pfn = uvm_physseg_get_start(i);
523: pfn < uvm_physseg_get_end(i);
524: pfn++) {
525: pg = PHYS_TO_VM_PAGE(ptoa(pfn));
526: md = VM_PAGE_TO_MD(pg);
527: mutex_init(&md->mdpg_pvlock, MUTEX_SPIN, IPL_VM);
528: }
529: }
530: }
531:
532: void
533: pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp)
534: {
535: *vstartp = virtual_avail;
536: *vendp = virtual_end;
537: }
538:
539: vaddr_t
540: pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp)
541: {
542: int npage;
543: paddr_t pa;
544: vaddr_t va;
545: psize_t bank_npage;
546: uvm_physseg_t bank;
547:
548: UVMHIST_FUNC(__func__);
549: UVMHIST_CALLED(pmaphist);
550:
551: UVMHIST_LOG(pmaphist, "size=%llu, *vstartp=%llx, *vendp=%llx",
552: size, *vstartp, *vendp, 0);
553:
554: size = round_page(size);
555: npage = atop(size);
556:
557: for (bank = uvm_physseg_get_first(); uvm_physseg_valid_p(bank);
558: bank = uvm_physseg_get_next(bank)) {
559:
560: bank_npage = uvm_physseg_get_avail_end(bank) -
561: uvm_physseg_get_avail_start(bank);
562: if (npage <= bank_npage)
563: break;
564: }
565:
1.23 maxv 566: if (!uvm_physseg_valid_p(bank)) {
567: panic("%s: no memory", __func__);
568: }
1.2 ryo 569:
570: /* Steal pages */
571: pa = ptoa(uvm_physseg_get_avail_start(bank));
572: va = AARCH64_PA_TO_KVA(pa);
573: uvm_physseg_unplug(atop(pa), npage);
574:
575: for (; npage > 0; npage--, pa += PAGE_SIZE)
576: pmap_zero_page(pa);
577:
578: return va;
1.1 matt 579: }
580:
581: void
582: pmap_reference(struct pmap *pm)
583: {
1.2 ryo 584: atomic_inc_uint(&pm->pm_refcnt);
1.1 matt 585: }
586:
1.36 ryo 587: paddr_t
1.40 ryo 588: pmap_alloc_pdp(struct pmap *pm, struct vm_page **pgp, int flags, bool waitok)
1.1 matt 589: {
1.2 ryo 590: paddr_t pa;
1.36 ryo 591: struct vm_page *pg;
1.2 ryo 592:
593: UVMHIST_FUNC(__func__);
594: UVMHIST_CALLED(pmaphist);
595:
596: if (uvm.page_init_done) {
1.40 ryo 597: int aflags = ((flags & PMAP_CANFAIL) ? 0 : UVM_PGA_USERESERVE) |
598: UVM_PGA_ZERO;
1.36 ryo 599: retry:
1.40 ryo 600: pg = uvm_pagealloc(NULL, 0, NULL, aflags);
1.36 ryo 601: if (pg == NULL) {
602: if (waitok) {
603: uvm_wait("pmap_alloc_pdp");
604: goto retry;
605: }
606: return POOL_PADDR_INVALID;
607: }
608:
609: TAILQ_INSERT_HEAD(&pm->pm_vmlist, pg, mdpage.mdpg_vmlist);
610: pg->flags &= ~PG_BUSY; /* never busy */
611: pg->wire_count = 1; /* max = 1 + Ln_ENTRIES = 513 */
1.2 ryo 612: pa = VM_PAGE_TO_PHYS(pg);
1.36 ryo 613: PMAP_COUNT(pdp_alloc);
1.2 ryo 614:
1.36 ryo 615: VM_PAGE_TO_MD(pg)->mdpg_ptep_parent = NULL;
1.2 ryo 616:
617: } else {
618: /* uvm_pageboot_alloc() returns AARCH64 KSEG address */
1.36 ryo 619: pg = NULL;
1.2 ryo 620: pa = AARCH64_KVA_TO_PA(
621: uvm_pageboot_alloc(Ln_TABLE_SIZE));
622: PMAP_COUNT(pdp_alloc_boot);
623: }
1.36 ryo 624: if (pgp != NULL)
625: *pgp = pg;
626:
627: UVMHIST_LOG(pmaphist, "pa=%llx, pg=%llx",
628: pa, pg, 0, 0);
629:
630: return pa;
631: }
1.2 ryo 632:
1.36 ryo 633: static void
634: pmap_free_pdp(struct pmap *pm, struct vm_page *pg)
635: {
636: TAILQ_REMOVE(&pm->pm_vmlist, pg, mdpage.mdpg_vmlist);
637: pg->flags |= PG_BUSY;
638: pg->wire_count = 0;
639: VM_MDPAGE_INIT(pg);
1.2 ryo 640:
1.36 ryo 641: uvm_pagefree(pg);
642: PMAP_COUNT(pdp_free);
1.1 matt 643: }
644:
1.40 ryo 645: /* free empty page table pages */
646: static int
647: _pmap_sweep_pdp(struct pmap *pm)
648: {
649: struct vm_page *pg, *tmp;
1.41 mrg 650: pd_entry_t *ptep_in_parent, opte __diagused;
1.40 ryo 651: paddr_t pa, pdppa;
652: int nsweep;
1.41 mrg 653: uint16_t wirecount __diagused;
1.40 ryo 654:
655: nsweep = 0;
656: TAILQ_FOREACH_SAFE(pg, &pm->pm_vmlist, mdpage.mdpg_vmlist, tmp) {
657: if (pg->wire_count != 1)
658: continue;
659:
660: pa = VM_PAGE_TO_PHYS(pg);
661: if (pa == pm->pm_l0table_pa)
662: continue;
663:
664: ptep_in_parent = VM_PAGE_TO_MD(pg)->mdpg_ptep_parent;
665: if (ptep_in_parent == NULL) {
666: /* no parent */
667: pmap_free_pdp(pm, pg);
668: nsweep++;
669: continue;
670: }
671:
672: /* unlink from parent */
673: opte = atomic_swap_64(ptep_in_parent, 0);
674: KASSERT(lxpde_valid(opte));
1.53 skrll 675: wirecount = atomic_add_32_nv(&pg->wire_count, -1); /* 1 -> 0 */
1.40 ryo 676: KASSERT(wirecount == 0);
677: pmap_free_pdp(pm, pg);
678: nsweep++;
679:
680: /* L3->L2->L1. no need for L0 */
681: pdppa = AARCH64_KVA_TO_PA(trunc_page((vaddr_t)ptep_in_parent));
682: if (pdppa == pm->pm_l0table_pa)
683: continue;
684:
685: pg = PHYS_TO_VM_PAGE(pdppa);
686: KASSERT(pg != NULL);
687: KASSERTMSG(pg->wire_count >= 1,
688: "wire_count=%d", pg->wire_count);
689: /* decrement wire_count of parent */
1.53 skrll 690: wirecount = atomic_add_32_nv(&pg->wire_count, -1);
1.40 ryo 691: KASSERTMSG(pg->wire_count <= (Ln_ENTRIES + 1),
692: "pm=%p[%d], pg=%p, wire_count=%d",
693: pm, pm->pm_asid, pg, pg->wire_count);
694: }
695: atomic_swap_uint(&pm->pm_idlepdp, 0);
696:
697: return nsweep;
698: }
699:
1.2 ryo 700: static void
701: _pmap_free_pdp_all(struct pmap *pm)
1.1 matt 702: {
1.2 ryo 703: struct vm_page *pg, *tmp;
704:
1.36 ryo 705: TAILQ_FOREACH_SAFE(pg, &pm->pm_vmlist, mdpage.mdpg_vmlist, tmp) {
706: pmap_free_pdp(pm, pg);
1.2 ryo 707: }
1.1 matt 708: }
709:
710: vaddr_t
711: pmap_growkernel(vaddr_t maxkvaddr)
712: {
1.2 ryo 713: UVMHIST_FUNC(__func__);
714: UVMHIST_CALLED(pmaphist);
715:
716: UVMHIST_LOG(pmaphist, "maxkvaddr=%llx, pmap_maxkvaddr=%llx",
717: maxkvaddr, pmap_maxkvaddr, 0, 0);
718:
1.54 ryo 719: kasan_shadow_map((void *)pmap_maxkvaddr,
720: (size_t)(maxkvaddr - pmap_maxkvaddr));
1.52 skrll 721:
1.2 ryo 722: pmap_maxkvaddr = maxkvaddr;
723:
1.1 matt 724: return maxkvaddr;
725: }
726:
1.2 ryo 727: bool
728: pmap_extract_coherency(struct pmap *pm, vaddr_t va, paddr_t *pap,
729: bool *coherencyp)
730: {
731: if (coherencyp)
732: *coherencyp = false;
733:
734: return pmap_extract(pm, va, pap);
735: }
736:
737: bool
738: pmap_extract(struct pmap *pm, vaddr_t va, paddr_t *pap)
739: {
1.36 ryo 740: pt_entry_t *ptep, pte;
1.2 ryo 741: paddr_t pa;
1.27 ryo 742: vsize_t blocksize = 0;
1.28 ryo 743: extern char __kernel_text[];
744: extern char _end[];
1.2 ryo 745:
1.28 ryo 746: if (IN_RANGE(va, (vaddr_t)__kernel_text, (vaddr_t)_end)) {
747: /* fast loookup */
748: pa = KERN_VTOPHYS(va);
749: } else if (IN_KSEG_ADDR(va)) {
750: /* fast loookup. should be used only if actually mapped? */
751: pa = AARCH64_KVA_TO_PA(va);
752: } else {
753: ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
754: if (ptep == NULL)
755: return false;
1.35 ryo 756: pte = *ptep;
757: if (!lxpde_valid(pte))
758: return false;
759: pa = lxpde_pa(pte) + (va & (blocksize - 1));
1.28 ryo 760: }
1.2 ryo 761:
1.27 ryo 762: if (pap != NULL)
1.2 ryo 763: *pap = pa;
1.27 ryo 764: return true;
1.2 ryo 765: }
766:
767: paddr_t
768: vtophys(vaddr_t va)
769: {
770: struct pmap *pm;
771: paddr_t pa;
772:
1.28 ryo 773: if (va & TTBR_SEL_VA)
774: pm = pmap_kernel();
775: else
1.2 ryo 776: pm = curlwp->l_proc->p_vmspace->vm_map.pmap;
1.28 ryo 777:
778: if (pmap_extract(pm, va, &pa) == false)
1.2 ryo 779: return VTOPHYS_FAILED;
780: return pa;
781: }
782:
1.36 ryo 783: /*
784: * return pointer of the pte. regardess of whether the entry is valid or not.
785: */
1.2 ryo 786: static pt_entry_t *
1.27 ryo 787: _pmap_pte_lookup_bs(struct pmap *pm, vaddr_t va, vsize_t *bs)
1.2 ryo 788: {
1.27 ryo 789: pt_entry_t *ptep;
790: pd_entry_t *l0, *l1, *l2, *l3;
791: pd_entry_t pde;
792: vsize_t blocksize;
793: unsigned int idx;
794:
795: if (((pm == pmap_kernel()) && ((va & TTBR_SEL_VA) == 0)) ||
796: ((pm != pmap_kernel()) && ((va & TTBR_SEL_VA) != 0))) {
797: blocksize = 0;
798: ptep = NULL;
799: goto done;
800: }
801:
802: /*
803: * traverse L0 -> L1 -> L2 -> L3
804: */
805: blocksize = L0_SIZE;
806: l0 = pm->pm_l0table;
807: idx = l0pde_index(va);
1.35 ryo 808: ptep = &l0[idx];
809: pde = *ptep;
810: if (!l0pde_valid(pde))
1.27 ryo 811: goto done;
812:
813: blocksize = L1_SIZE;
814: l1 = (pd_entry_t *)AARCH64_PA_TO_KVA(l0pde_pa(pde));
815: idx = l1pde_index(va);
1.35 ryo 816: ptep = &l1[idx];
817: pde = *ptep;
818: if (!l1pde_valid(pde) || l1pde_is_block(pde))
1.27 ryo 819: goto done;
820:
821: blocksize = L2_SIZE;
822: l2 = (pd_entry_t *)AARCH64_PA_TO_KVA(l1pde_pa(pde));
823: idx = l2pde_index(va);
1.35 ryo 824: ptep = &l2[idx];
825: pde = *ptep;
826: if (!l2pde_valid(pde) || l2pde_is_block(pde))
1.27 ryo 827: goto done;
1.2 ryo 828:
1.27 ryo 829: blocksize = L3_SIZE;
830: l3 = (pd_entry_t *)AARCH64_PA_TO_KVA(l2pde_pa(pde));
831: idx = l3pte_index(va);
832: ptep = &l3[idx];
1.2 ryo 833:
1.27 ryo 834: done:
835: if (bs != NULL)
836: *bs = blocksize;
837: return ptep;
838: }
1.2 ryo 839:
1.27 ryo 840: static pt_entry_t *
841: _pmap_pte_lookup_l3(struct pmap *pm, vaddr_t va)
842: {
843: pt_entry_t *ptep;
844: vsize_t blocksize = 0;
845:
846: ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
847: if ((ptep != NULL) && (blocksize == L3_SIZE))
1.2 ryo 848: return ptep;
849:
850: return NULL;
851: }
852:
1.29 ryo 853: void
854: pmap_icache_sync_range(pmap_t pm, vaddr_t sva, vaddr_t eva)
855: {
1.35 ryo 856: pt_entry_t *ptep = NULL, pte;
1.29 ryo 857: vaddr_t va;
858: vsize_t blocksize = 0;
859:
860: pm_lock(pm);
861:
1.35 ryo 862: for (va = sva; va < eva; va = (va + blocksize) & ~(blocksize - 1)) {
863: /* va is belong to the same L3 table as before? */
864: if ((blocksize == L3_SIZE) && ((va & L3INDEXMASK) != 0)) {
865: ptep++;
866: } else {
867: ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
868: if (ptep == NULL)
869: break;
870: }
871:
872: pte = *ptep;
873: if (lxpde_valid(pte)) {
1.29 ryo 874: vaddr_t eob = (va + blocksize) & ~(blocksize - 1);
875: vsize_t len = ulmin(eva, eob - va);
876:
877: if (l3pte_writable(pte)) {
878: cpu_icache_sync_range(va, len);
879: } else {
880: /*
881: * change to writable temporally
882: * to do cpu_icache_sync_range()
883: */
884: pt_entry_t opte = pte;
885: pte = pte & ~(LX_BLKPAG_AF|LX_BLKPAG_AP);
886: pte |= (LX_BLKPAG_AF|LX_BLKPAG_AP_RW);
887: atomic_swap_64(ptep, pte);
888: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
889: cpu_icache_sync_range(va, len);
890: atomic_swap_64(ptep, opte);
891: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
892: }
893: }
894: }
895:
896: pm_unlock(pm);
897: }
898:
1.48 maya 899: /*
900: * Routine: pmap_procwr
901: *
902: * Function:
903: * Synchronize caches corresponding to [addr, addr+len) in p.
904: *
905: */
906: void
907: pmap_procwr(struct proc *p, vaddr_t va, int len)
908: {
909:
910: /* We only need to do anything if it is the current process. */
911: if (p == curproc)
912: cpu_icache_sync_range(va, len);
913: }
914:
1.2 ryo 915: static pt_entry_t
1.18 ryo 916: _pmap_pte_adjust_prot(pt_entry_t pte, vm_prot_t prot, vm_prot_t protmask,
917: bool user)
1.1 matt 918: {
1.2 ryo 919: vm_prot_t masked;
1.18 ryo 920: pt_entry_t xn;
1.2 ryo 921:
922: masked = prot & protmask;
923: pte &= ~(LX_BLKPAG_OS_RWMASK|LX_BLKPAG_AF|LX_BLKPAG_AP);
924:
925: /* keep prot for ref/mod emulation */
926: switch (prot & (VM_PROT_READ|VM_PROT_WRITE)) {
927: case 0:
928: default:
929: break;
930: case VM_PROT_READ:
931: pte |= LX_BLKPAG_OS_READ;
932: break;
933: case VM_PROT_WRITE:
934: case VM_PROT_READ|VM_PROT_WRITE:
935: pte |= (LX_BLKPAG_OS_READ|LX_BLKPAG_OS_WRITE);
936: break;
937: }
938:
939: switch (masked & (VM_PROT_READ|VM_PROT_WRITE)) {
940: case 0:
941: default:
942: /* cannot access due to No LX_BLKPAG_AF */
943: pte |= LX_BLKPAG_AP_RO;
944: break;
945: case VM_PROT_READ:
946: /* actual permission of pte */
947: pte |= LX_BLKPAG_AF;
948: pte |= LX_BLKPAG_AP_RO;
949: break;
950: case VM_PROT_WRITE:
951: case VM_PROT_READ|VM_PROT_WRITE:
952: /* actual permission of pte */
953: pte |= LX_BLKPAG_AF;
954: pte |= LX_BLKPAG_AP_RW;
955: break;
956: }
957:
1.18 ryo 958: /* executable for kernel or user? first set never exec both */
959: pte |= (LX_BLKPAG_UXN|LX_BLKPAG_PXN);
960: /* and either to executable */
961: xn = user ? LX_BLKPAG_UXN : LX_BLKPAG_PXN;
962: if (prot & VM_PROT_EXECUTE)
1.35 ryo 963: pte &= ~xn;
1.2 ryo 964:
965: return pte;
966: }
967:
968: static pt_entry_t
969: _pmap_pte_adjust_cacheflags(pt_entry_t pte, u_int flags)
970: {
971:
972: pte &= ~LX_BLKPAG_ATTR_MASK;
973:
1.58 jmcneill 974: switch (flags & (PMAP_CACHE_MASK|PMAP_DEV_MASK)) {
975: case PMAP_DEV_SO ... PMAP_DEV_SO | PMAP_CACHE_MASK:
976: pte |= LX_BLKPAG_ATTR_DEVICE_MEM_SO; /* Device-nGnRnE */
977: break;
1.2 ryo 978: case PMAP_DEV ... PMAP_DEV | PMAP_CACHE_MASK:
1.57 jmcneill 979: pte |= LX_BLKPAG_ATTR_DEVICE_MEM; /* Device-nGnRE */
1.2 ryo 980: break;
981: case PMAP_NOCACHE:
982: case PMAP_NOCACHE_OVR:
983: case PMAP_WRITE_COMBINE:
984: pte |= LX_BLKPAG_ATTR_NORMAL_NC; /* only no-cache */
985: break;
986: case PMAP_WRITE_BACK:
987: case 0:
988: default:
989: pte |= LX_BLKPAG_ATTR_NORMAL_WB;
990: break;
991: }
992:
993: return pte;
994: }
995:
1.14 ryo 996: static struct pv_entry *
1.2 ryo 997: _pmap_remove_pv(struct vm_page *pg, struct pmap *pm, vaddr_t va, pt_entry_t pte)
998: {
999: struct vm_page_md *md;
1000: struct pv_entry *pv;
1001:
1002: UVMHIST_FUNC(__func__);
1003: UVMHIST_CALLED(pmaphist);
1004:
1005: UVMHIST_LOG(pmaphist, "pg=%p, pm=%p, va=%llx, pte=%llx",
1006: pg, pm, va, pte);
1007:
1008: md = VM_PAGE_TO_MD(pg);
1009:
1010: TAILQ_FOREACH(pv, &md->mdpg_pvhead, pv_link) {
1011: if ((pm == pv->pv_pmap) && (va == pv->pv_va)) {
1.5 ryo 1012: TAILQ_REMOVE(&md->mdpg_pvhead, pv, pv_link);
1013: PMAP_COUNT(pv_remove);
1.2 ryo 1014: break;
1015: }
1016: }
1.5 ryo 1017: #ifdef PMAPCOUNTERS
1018: if (pv == NULL) {
1019: PMAP_COUNT(pv_remove_nopv);
1020: }
1021: #endif
1.2 ryo 1022:
1.14 ryo 1023: return pv;
1.2 ryo 1024: }
1025:
1026: #if defined(PMAP_PV_DEBUG) || defined(DDB)
1027:
1028: static char *
1029: str_vmflags(uint32_t flags)
1030: {
1031: static int idx = 0;
1032: static char buf[4][32]; /* XXX */
1033: char *p;
1034:
1035: p = buf[idx];
1036: idx = (idx + 1) & 3;
1037:
1038: p[0] = (flags & VM_PROT_READ) ? 'R' : '-';
1039: p[1] = (flags & VM_PROT_WRITE) ? 'W' : '-';
1040: p[2] = (flags & VM_PROT_EXECUTE) ? 'X' : '-';
1041: if (flags & PMAP_WIRED)
1042: memcpy(&p[3], ",WIRED\0", 7);
1043: else
1044: p[3] = '\0';
1045:
1046: return p;
1047: }
1048:
1049: static void
1.38 ryo 1050: pg_dump(struct vm_page *pg, void (*pr)(const char *, ...) __printflike(1, 2))
1.2 ryo 1051: {
1052: pr("pg=%p\n", pg);
1053: pr(" pg->uanon = %p\n", pg->uanon);
1054: pr(" pg->uobject = %p\n", pg->uobject);
1055: pr(" pg->offset = %zu\n", pg->offset);
1056: pr(" pg->flags = %u\n", pg->flags);
1057: pr(" pg->loan_count = %u\n", pg->loan_count);
1058: pr(" pg->wire_count = %u\n", pg->wire_count);
1059: pr(" pg->pqflags = %u\n", pg->pqflags);
1.51 ad 1060: pr(" pg->phys_addr = %016lx\n", VM_PAGE_TO_PHYS(pg));
1.2 ryo 1061: }
1062:
1063: static void
1.38 ryo 1064: pv_dump(struct vm_page_md *md, void (*pr)(const char *, ...) __printflike(1, 2))
1.2 ryo 1065: {
1066: struct pv_entry *pv;
1067: int i;
1068:
1069: i = 0;
1070:
1071: pr("md=%p\n", md);
1072: pr(" md->mdpg_flags=%08x %s\n", md->mdpg_flags,
1073: str_vmflags(md->mdpg_flags));
1074:
1075: TAILQ_FOREACH(pv, &md->mdpg_pvhead, pv_link) {
1076: pr(" pv[%d] pv=%p\n",
1077: i, pv);
1.35 ryo 1078: pr(" pv[%d].pv_pmap = %p (asid=%d)\n",
1.2 ryo 1079: i, pv->pv_pmap, pv->pv_pmap->pm_asid);
1.35 ryo 1080: pr(" pv[%d].pv_va = %016lx (color=%d)\n",
1.2 ryo 1081: i, pv->pv_va, _pmap_color(pv->pv_va));
1.35 ryo 1082: pr(" pv[%d].pv_pa = %016lx (color=%d)\n",
1.2 ryo 1083: i, pv->pv_pa, _pmap_color(pv->pv_pa));
1.35 ryo 1084: pr(" pv[%d].pv_ptep = %p\n",
1085: i, pv->pv_ptep);
1.2 ryo 1086: i++;
1087: }
1088: }
1089: #endif /* PMAP_PV_DEBUG & DDB */
1090:
1091: static int
1.35 ryo 1092: _pmap_enter_pv(struct vm_page *pg, struct pmap *pm, struct pv_entry **pvp,
1093: vaddr_t va, pt_entry_t *ptep, paddr_t pa, u_int flags)
1.2 ryo 1094: {
1095: struct vm_page_md *md;
1096: struct pv_entry *pv;
1097:
1098: UVMHIST_FUNC(__func__);
1099: UVMHIST_CALLED(pmaphist);
1100:
1101: UVMHIST_LOG(pmaphist, "pg=%p, pm=%p, va=%llx, pa=%llx", pg, pm, va, pa);
1102: UVMHIST_LOG(pmaphist, "ptep=%p, flags=%08x", ptep, flags, 0, 0);
1103:
1104: md = VM_PAGE_TO_MD(pg);
1105:
1106: /* pv is already registered? */
1107: TAILQ_FOREACH(pv, &md->mdpg_pvhead, pv_link) {
1108: if ((pm == pv->pv_pmap) && (va == pv->pv_va)) {
1109: break;
1110: }
1111: }
1112:
1.3 ryo 1113: if (pv == NULL) {
1.12 ryo 1114: /*
1115: * create and link new pv.
1116: * pv is already allocated at beginning of _pmap_enter().
1117: */
1118: pv = *pvp;
1.2 ryo 1119: if (pv == NULL)
1120: return ENOMEM;
1.12 ryo 1121: *pvp = NULL;
1.2 ryo 1122:
1123: pv->pv_pmap = pm;
1124: pv->pv_va = va;
1125: pv->pv_pa = pa;
1126: pv->pv_ptep = ptep;
1127:
1128: TAILQ_INSERT_HEAD(&md->mdpg_pvhead, pv, pv_link);
1129: PMAP_COUNT(pv_enter);
1130:
1131: #ifdef PMAP_PV_DEBUG
1132: if (!TAILQ_EMPTY(&md->mdpg_pvhead)){
1133: printf("pv %p alias added va=%016lx -> pa=%016lx\n",
1134: pv, va, pa);
1135: pv_dump(md, printf);
1136: }
1137: #endif
1138: }
1.35 ryo 1139:
1.1 matt 1140: return 0;
1141: }
1142:
1143: void
1.2 ryo 1144: pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1.1 matt 1145: {
1.2 ryo 1146: int s;
1147:
1148: s = splvm();
1149: _pmap_enter(pmap_kernel(), va, pa, prot, flags | PMAP_WIRED, true);
1150: splx(s);
1.1 matt 1151: }
1152:
1153: void
1.2 ryo 1154: pmap_kremove(vaddr_t va, vsize_t size)
1.1 matt 1155: {
1.2 ryo 1156: struct pmap *kpm = pmap_kernel();
1.3 ryo 1157: int s;
1.2 ryo 1158:
1159: UVMHIST_FUNC(__func__);
1160: UVMHIST_CALLED(pmaphist);
1161:
1162: UVMHIST_LOG(pmaphist, "va=%llx, size=%llx", va, size, 0, 0);
1163:
1164: KDASSERT((va & PGOFSET) == 0);
1165: KDASSERT((size & PGOFSET) == 0);
1166:
1.28 ryo 1167: KDASSERT(!IN_KSEG_ADDR(va));
1168: KDASSERT(IN_RANGE(va, VM_MIN_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS));
1.2 ryo 1169:
1.3 ryo 1170: s = splvm();
1.35 ryo 1171: pm_lock(kpm);
1172: _pmap_remove(kpm, va, va + size, true, NULL);
1173: pm_unlock(kpm);
1.3 ryo 1174: splx(s);
1.2 ryo 1175: }
1176:
1177: static void
1178: _pmap_protect_pv(struct vm_page *pg, struct pv_entry *pv, vm_prot_t prot)
1179: {
1180: pt_entry_t *ptep, pte;
1181: vm_prot_t pteprot;
1182: uint32_t mdattr;
1.18 ryo 1183: const bool user = (pv->pv_pmap != pmap_kernel());
1.2 ryo 1184:
1185: UVMHIST_FUNC(__func__);
1186: UVMHIST_CALLED(pmaphist);
1187:
1188: UVMHIST_LOG(pmaphist, "pg=%p, pv=%p, prot=%08x", pg, pv, prot, 0);
1189:
1190: /* get prot mask from referenced/modified */
1191: mdattr = VM_PAGE_TO_MD(pg)->mdpg_flags &
1192: (VM_PROT_READ | VM_PROT_WRITE);
1193:
1.11 ryo 1194: pm_lock(pv->pv_pmap);
1.2 ryo 1195:
1196: ptep = pv->pv_ptep;
1197: pte = *ptep;
1198:
1199: /* get prot mask from pte */
1200: pteprot = 0;
1.3 ryo 1201: if (pte & LX_BLKPAG_AF)
1.2 ryo 1202: pteprot |= VM_PROT_READ;
1.3 ryo 1203: if ((pte & LX_BLKPAG_AP) == LX_BLKPAG_AP_RW)
1.2 ryo 1204: pteprot |= VM_PROT_WRITE;
1.18 ryo 1205: if (l3pte_executable(pte, user))
1.2 ryo 1206: pteprot |= VM_PROT_EXECUTE;
1207:
1208: /* new prot = prot & pteprot & mdattr */
1.18 ryo 1209: pte = _pmap_pte_adjust_prot(pte, prot & pteprot, mdattr, user);
1.2 ryo 1210: atomic_swap_64(ptep, pte);
1.19 ryo 1211: AARCH64_TLBI_BY_ASID_VA(pv->pv_pmap->pm_asid, pv->pv_va, true);
1.2 ryo 1212:
1.11 ryo 1213: pm_unlock(pv->pv_pmap);
1.1 matt 1214: }
1215:
1216: void
1217: pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
1218: {
1.35 ryo 1219: pt_entry_t *ptep = NULL, pte;
1.2 ryo 1220: vaddr_t va;
1.35 ryo 1221: vsize_t blocksize = 0;
1.18 ryo 1222: const bool user = (pm != pmap_kernel());
1.2 ryo 1223:
1224: KASSERT((prot & VM_PROT_READ) || !(prot & VM_PROT_WRITE));
1225:
1226: UVMHIST_FUNC(__func__);
1227: UVMHIST_CALLED(pmaphist);
1228:
1229: UVMHIST_LOG(pmaphist, "pm=%p, sva=%016lx, eva=%016lx, prot=%08x",
1230: pm, sva, eva, prot);
1231:
1.28 ryo 1232: KASSERT_PM_ADDR(pm, sva);
1.2 ryo 1233: KASSERT(!IN_KSEG_ADDR(sva));
1234:
1.61 ! ryo 1235: /* PROT_EXEC requires implicit PROT_READ */
! 1236: if (prot & VM_PROT_EXECUTE)
! 1237: prot |= VM_PROT_READ;
! 1238:
1.2 ryo 1239: if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
1240: PMAP_COUNT(protect_remove_fallback);
1241: pmap_remove(pm, sva, eva);
1242: return;
1243: }
1244: PMAP_COUNT(protect);
1245:
1246: KDASSERT((sva & PAGE_MASK) == 0);
1247: KDASSERT((eva & PAGE_MASK) == 0);
1248:
1.11 ryo 1249: pm_lock(pm);
1.2 ryo 1250:
1.35 ryo 1251: for (va = sva; va < eva; va = (va + blocksize) & ~(blocksize - 1)) {
1.13 ryo 1252: #ifdef UVMHIST
1253: pt_entry_t opte;
1254: #endif
1.2 ryo 1255: struct vm_page *pg;
1256: paddr_t pa;
1257: uint32_t mdattr;
1258: bool executable;
1259:
1.35 ryo 1260: /* va is belong to the same L3 table as before? */
1261: if ((blocksize == L3_SIZE) && ((va & L3INDEXMASK) != 0))
1262: ptep++;
1263: else
1264: ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
1.2 ryo 1265:
1266: pte = *ptep;
1.35 ryo 1267: if (!lxpde_valid(pte)) {
1.2 ryo 1268: PMAP_COUNT(protect_none);
1269: continue;
1270: }
1271:
1.35 ryo 1272: pa = lxpde_pa(pte);
1.2 ryo 1273: pg = PHYS_TO_VM_PAGE(pa);
1274:
1275: if (pg != NULL) {
1.3 ryo 1276: /* get prot mask from referenced/modified */
1.2 ryo 1277: mdattr = VM_PAGE_TO_MD(pg)->mdpg_flags &
1278: (VM_PROT_READ | VM_PROT_WRITE);
1279: PMAP_COUNT(protect_managed);
1280: } else {
1281: /* unmanaged page */
1282: mdattr = VM_PROT_ALL;
1283: PMAP_COUNT(protect_unmanaged);
1284: }
1285:
1.13 ryo 1286: #ifdef UVMHIST
1287: opte = pte;
1288: #endif
1.18 ryo 1289: executable = l3pte_executable(pte, user);
1290: pte = _pmap_pte_adjust_prot(pte, prot, mdattr, user);
1.13 ryo 1291:
1292: if (!executable && (prot & VM_PROT_EXECUTE)) {
1293: /* non-exec -> exec */
1.15 ryo 1294: UVMHIST_LOG(pmaphist, "icache_sync: "
1295: "pm=%p, va=%016lx, pte: %016lx -> %016lx",
1.13 ryo 1296: pm, va, opte, pte);
1.15 ryo 1297: if (!l3pte_writable(pte)) {
1.19 ryo 1298: PTE_ICACHE_SYNC_PAGE(pte, ptep, pm, va, true);
1.15 ryo 1299: atomic_swap_64(ptep, pte);
1.19 ryo 1300: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
1.15 ryo 1301: } else {
1302: atomic_swap_64(ptep, pte);
1.19 ryo 1303: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
1.15 ryo 1304: cpu_icache_sync_range(va, PAGE_SIZE);
1305: }
1306: } else {
1307: atomic_swap_64(ptep, pte);
1.19 ryo 1308: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
1.15 ryo 1309: }
1.2 ryo 1310: }
1311:
1.11 ryo 1312: pm_unlock(pm);
1.1 matt 1313: }
1314:
1315: void
1.2 ryo 1316: pmap_activate(struct lwp *l)
1317: {
1318: struct pmap *pm = l->l_proc->p_vmspace->vm_map.pmap;
1.47 jmcneill 1319: uint64_t ttbr0, tcr;
1.2 ryo 1320:
1321: UVMHIST_FUNC(__func__);
1322: UVMHIST_CALLED(pmaphist);
1323:
1324: if (pm == pmap_kernel())
1325: return;
1326: if (l != curlwp)
1327: return;
1328:
1329: KASSERT(pm->pm_l0table != NULL);
1330:
1331: UVMHIST_LOG(pmaphist, "lwp=%p (pid=%d)", l, l->l_proc->p_pid, 0, 0);
1332:
1.47 jmcneill 1333: /* Disable translation table walks using TTBR0 */
1334: tcr = reg_tcr_el1_read();
1335: reg_tcr_el1_write(tcr | TCR_EPD0);
1336: __asm __volatile("isb" ::: "memory");
1337:
1.2 ryo 1338: /* XXX */
1339: CTASSERT(PID_MAX <= 65535); /* 16bit ASID */
1340: if (pm->pm_asid == -1)
1341: pm->pm_asid = l->l_proc->p_pid;
1342:
1343: ttbr0 = ((uint64_t)pm->pm_asid << 48) | pm->pm_l0table_pa;
1.34 ryo 1344: cpu_set_ttbr0(ttbr0);
1.2 ryo 1345:
1.47 jmcneill 1346: /* Re-enable translation table walks using TTBR0 */
1347: tcr = reg_tcr_el1_read();
1348: reg_tcr_el1_write(tcr & ~TCR_EPD0);
1349: __asm __volatile("isb" ::: "memory");
1350:
1.2 ryo 1351: pm->pm_activated = true;
1352:
1353: PMAP_COUNT(activate);
1354: }
1355:
1356: void
1357: pmap_deactivate(struct lwp *l)
1.1 matt 1358: {
1.2 ryo 1359: struct pmap *pm = l->l_proc->p_vmspace->vm_map.pmap;
1.47 jmcneill 1360: uint64_t tcr;
1.2 ryo 1361:
1362: UVMHIST_FUNC(__func__);
1363: UVMHIST_CALLED(pmaphist);
1364:
1365: if (pm == pmap_kernel())
1366: return;
1367:
1368: UVMHIST_LOG(pmaphist, "lwp=%p, asid=%d", l, pm->pm_asid, 0, 0);
1369:
1.47 jmcneill 1370: /* Disable translation table walks using TTBR0 */
1371: tcr = reg_tcr_el1_read();
1372: reg_tcr_el1_write(tcr | TCR_EPD0);
1373: __asm __volatile("isb" ::: "memory");
1374:
1.2 ryo 1375: /* XXX */
1376: pm->pm_activated = false;
1377:
1378: PMAP_COUNT(deactivate);
1.1 matt 1379: }
1380:
1.2 ryo 1381: struct pmap *
1382: pmap_create(void)
1.1 matt 1383: {
1.2 ryo 1384: struct pmap *pm;
1385:
1386: UVMHIST_FUNC(__func__);
1387: UVMHIST_CALLED(pmaphist);
1388:
1389: pm = pool_cache_get(&_pmap_cache, PR_WAITOK);
1390: memset(pm, 0, sizeof(*pm));
1391: pm->pm_refcnt = 1;
1.40 ryo 1392: pm->pm_idlepdp = 0;
1.2 ryo 1393: pm->pm_asid = -1;
1.36 ryo 1394: TAILQ_INIT(&pm->pm_vmlist);
1.6 ryo 1395: mutex_init(&pm->pm_lock, MUTEX_DEFAULT, IPL_VM);
1.39 ryo 1396:
1.40 ryo 1397: pm->pm_l0table_pa = pmap_alloc_pdp(pm, NULL, 0, true);
1.36 ryo 1398: KASSERT(pm->pm_l0table_pa != POOL_PADDR_INVALID);
1399: pm->pm_l0table = (pd_entry_t *)AARCH64_PA_TO_KVA(pm->pm_l0table_pa);
1.2 ryo 1400: KASSERT(((vaddr_t)pm->pm_l0table & (PAGE_SIZE - 1)) == 0);
1401:
1402: UVMHIST_LOG(pmaphist, "pm=%p, pm_l0table=%016lx, pm_l0table_pa=%016lx",
1403: pm, pm->pm_l0table, pm->pm_l0table_pa, 0);
1404:
1405: PMAP_COUNT(create);
1406: return pm;
1.1 matt 1407: }
1408:
1409: void
1.2 ryo 1410: pmap_destroy(struct pmap *pm)
1411: {
1412: unsigned int refcnt;
1413:
1414: UVMHIST_FUNC(__func__);
1415: UVMHIST_CALLED(pmaphist);
1416:
1417: UVMHIST_LOG(pmaphist,
1418: "pm=%p, pm_l0table=%016lx, pm_l0table_pa=%016lx, refcnt=%d",
1419: pm, pm->pm_l0table, pm->pm_l0table_pa, pm->pm_refcnt);
1420:
1421: if (pm == NULL)
1422: return;
1423:
1424: if (pm == pmap_kernel())
1425: panic("cannot destroy kernel pmap");
1426:
1427: refcnt = atomic_dec_uint_nv(&pm->pm_refcnt);
1428: if (refcnt > 0)
1429: return;
1430:
1431: aarch64_tlbi_by_asid(pm->pm_asid);
1432:
1433: _pmap_free_pdp_all(pm);
1434: mutex_destroy(&pm->pm_lock);
1.35 ryo 1435:
1.2 ryo 1436: pool_cache_put(&_pmap_cache, pm);
1437:
1438: PMAP_COUNT(destroy);
1439: }
1440:
1.36 ryo 1441: static inline void
1442: _pmap_pdp_setparent(struct pmap *pm, struct vm_page *pg, pt_entry_t *ptep)
1443: {
1444: if ((pm != pmap_kernel()) && (pg != NULL))
1445: VM_PAGE_TO_MD(pg)->mdpg_ptep_parent = ptep;
1446: }
1447:
1448: /*
1449: * increment reference counter of the page descriptor page.
1450: * the reference counter should be equal to
1451: * 1 + num of valid entries the page has.
1452: */
1453: static inline void
1454: _pmap_pdp_addref(struct pmap *pm, paddr_t pdppa, struct vm_page *pdppg_hint)
1455: {
1456: struct vm_page *pg;
1457:
1458: /* kernel L0-L3 page will be never freed */
1459: if (pm == pmap_kernel())
1460: return;
1461: /* no need for L0 page */
1462: if (pm->pm_l0table_pa == pdppa)
1463: return;
1464:
1465: pg = pdppg_hint;
1466: if (pg == NULL)
1467: pg = PHYS_TO_VM_PAGE(pdppa);
1468: KASSERT(pg != NULL);
1469:
1.53 skrll 1470: CTASSERT(sizeof(pg->wire_count) == sizeof(uint32_t));
1471: atomic_add_32(&pg->wire_count, 1);
1.36 ryo 1472:
1473: KASSERTMSG(pg->wire_count <= (Ln_ENTRIES + 1),
1474: "pg=%p, wire_count=%d", pg, pg->wire_count);
1475: }
1476:
1477: /*
1478: * decrement reference counter of the page descriptr page.
1479: * if reference counter is 1(=empty), pages will be freed, and return true.
1480: * otherwise return false.
1481: * kernel page, or L0 page descriptor page will be never freed.
1482: */
1483: static bool
1484: _pmap_pdp_delref(struct pmap *pm, paddr_t pdppa, bool do_free_pdp)
1485: {
1486: struct vm_page *pg;
1487: bool removed;
1488: uint16_t wirecount;
1489:
1490: /* kernel L0-L3 page will be never freed */
1491: if (pm == pmap_kernel())
1492: return false;
1493: /* no need for L0 page */
1494: if (pm->pm_l0table_pa == pdppa)
1495: return false;
1496:
1497: pg = PHYS_TO_VM_PAGE(pdppa);
1498: KASSERT(pg != NULL);
1499:
1.53 skrll 1500: wirecount = atomic_add_32_nv(&pg->wire_count, -1);
1.36 ryo 1501:
1.40 ryo 1502: if (!do_free_pdp) {
1503: /*
1504: * pm_idlepdp is counted by only pmap_page_protect() with
1505: * VM_PROT_NONE. it is not correct because without considering
1506: * pmap_enter(), but useful hint to just sweep.
1507: */
1508: if (wirecount == 1)
1509: atomic_inc_uint(&pm->pm_idlepdp);
1.36 ryo 1510: return false;
1.40 ryo 1511: }
1.36 ryo 1512:
1513: /* if no reference, free pdp */
1514: removed = false;
1515: while (wirecount == 1) {
1.41 mrg 1516: pd_entry_t *ptep_in_parent, opte __diagused;
1.36 ryo 1517: ptep_in_parent = VM_PAGE_TO_MD(pg)->mdpg_ptep_parent;
1518: if (ptep_in_parent == NULL) {
1519: /* no parent */
1520: pmap_free_pdp(pm, pg);
1521: removed = true;
1522: break;
1523: }
1524:
1525: /* unlink from parent */
1526: opte = atomic_swap_64(ptep_in_parent, 0);
1527: KASSERT(lxpde_valid(opte));
1.53 skrll 1528: wirecount = atomic_add_32_nv(&pg->wire_count, -1); /* 1 -> 0 */
1.36 ryo 1529: KASSERT(wirecount == 0);
1530: pmap_free_pdp(pm, pg);
1531: removed = true;
1532:
1533: /* L3->L2->L1. no need for L0 */
1534: pdppa = AARCH64_KVA_TO_PA(trunc_page((vaddr_t)ptep_in_parent));
1535: if (pdppa == pm->pm_l0table_pa)
1536: break;
1537:
1538: pg = PHYS_TO_VM_PAGE(pdppa);
1539: KASSERT(pg != NULL);
1540: KASSERTMSG(pg->wire_count >= 1,
1541: "wire_count=%d", pg->wire_count);
1542: /* decrement wire_count of parent */
1.53 skrll 1543: wirecount = atomic_add_32_nv(&pg->wire_count, -1);
1.36 ryo 1544: KASSERTMSG(pg->wire_count <= (Ln_ENTRIES + 1),
1545: "pm=%p[%d], pg=%p, wire_count=%d",
1546: pm, pm->pm_asid, pg, pg->wire_count);
1547: }
1548:
1549: return removed;
1550: }
1551:
1.2 ryo 1552: static int
1553: _pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot,
1554: u_int flags, bool kenter)
1555: {
1.40 ryo 1556: struct vm_page *pg, *pgs[2], *pdppg, *pdppg0;
1.14 ryo 1557: struct pv_entry *spv, *opv = NULL;
1.2 ryo 1558: pd_entry_t pde;
1.39 ryo 1559: pt_entry_t attr, pte, opte, *ptep;
1.2 ryo 1560: pd_entry_t *l0, *l1, *l2, *l3;
1.36 ryo 1561: paddr_t pdppa, pdppa0;
1.2 ryo 1562: uint32_t mdattr;
1563: unsigned int idx;
1564: int error = 0;
1565: const bool user = (pm != pmap_kernel());
1.39 ryo 1566: bool need_sync_icache, need_update_pv;
1.19 ryo 1567: bool l3only = true;
1.2 ryo 1568:
1569: UVMHIST_FUNC(__func__);
1570: UVMHIST_CALLED(pmaphist);
1571:
1572: UVMHIST_LOG(pmaphist, "pm=%p, kentermode=%d", pm, kenter, 0, 0);
1573: UVMHIST_LOG(pmaphist, "va=%016lx, pa=%016lx, prot=%08x, flags=%08x",
1574: va, pa, prot, flags);
1575:
1.28 ryo 1576: KASSERT_PM_ADDR(pm, va);
1577: KASSERT(!IN_KSEG_ADDR(va));
1.2 ryo 1578:
1579: #ifdef PMAPCOUNTERS
1580: PMAP_COUNT(mappings);
1581: if (_pmap_color(va) == _pmap_color(pa)) {
1582: if (user) {
1583: PMAP_COUNT(user_mappings);
1584: } else {
1585: PMAP_COUNT(kern_mappings);
1586: }
1587: } else if (flags & PMAP_WIRED) {
1588: if (user) {
1589: PMAP_COUNT(user_mappings_bad_wired);
1590: } else {
1591: PMAP_COUNT(kern_mappings_bad_wired);
1592: }
1593: } else {
1594: if (user) {
1595: PMAP_COUNT(user_mappings_bad);
1596: } else {
1597: PMAP_COUNT(kern_mappings_bad);
1598: }
1599: }
1600: #endif
1601:
1.3 ryo 1602: if (kenter)
1603: pg = NULL;
1604: else
1605: pg = PHYS_TO_VM_PAGE(pa);
1606:
1.12 ryo 1607: if (pg != NULL) {
1608: PMAP_COUNT(managed_mappings);
1609: /*
1610: * allocate pv in advance of pm_lock() to avoid locking myself.
1611: * pool_cache_get() may call pmap_kenter() internally.
1612: */
1613: spv = pool_cache_get(&_pmap_pv_pool, PR_NOWAIT);
1.39 ryo 1614: need_update_pv = true;
1.12 ryo 1615: } else {
1.2 ryo 1616: PMAP_COUNT(unmanaged_mappings);
1.12 ryo 1617: spv = NULL;
1.39 ryo 1618: need_update_pv = false;
1.2 ryo 1619: }
1620:
1.12 ryo 1621: pm_lock(pm);
1.2 ryo 1622:
1.40 ryo 1623: if (pm->pm_idlepdp >= PDPSWEEP_TRIGGER) {
1624: if (_pmap_sweep_pdp(pm) != 0) {
1625: /* several L1-L3 page table pages have been freed */
1626: aarch64_tlbi_by_asid(pm->pm_asid);
1627: }
1628: }
1629:
1.2 ryo 1630: /*
1631: * traverse L0 -> L1 -> L2 -> L3 table with growing pdp if needed.
1632: */
1633: l0 = pm->pm_l0table;
1634:
1635: idx = l0pde_index(va);
1636: pde = l0[idx];
1637: if (!l0pde_valid(pde)) {
1.36 ryo 1638: /* no need to increment L0 occupancy. L0 page never freed */
1.40 ryo 1639: pdppa = pmap_alloc_pdp(pm, &pdppg, flags, false); /* L1 pdp */
1.36 ryo 1640: if (pdppa == POOL_PADDR_INVALID) {
1641: if (flags & PMAP_CANFAIL) {
1642: error = ENOMEM;
1.39 ryo 1643: goto fail0;
1.36 ryo 1644: }
1.39 ryo 1645: pm_unlock(pm);
1.36 ryo 1646: panic("%s: cannot allocate L1 table", __func__);
1647: }
1.2 ryo 1648: atomic_swap_64(&l0[idx], pdppa | L0_TABLE);
1.36 ryo 1649: _pmap_pdp_setparent(pm, pdppg, &l0[idx]);
1.19 ryo 1650: l3only = false;
1.2 ryo 1651: } else {
1652: pdppa = l0pde_pa(pde);
1.36 ryo 1653: pdppg = NULL;
1.2 ryo 1654: }
1.9 christos 1655: l1 = (void *)AARCH64_PA_TO_KVA(pdppa);
1.2 ryo 1656:
1657: idx = l1pde_index(va);
1658: pde = l1[idx];
1659: if (!l1pde_valid(pde)) {
1.36 ryo 1660: pdppa0 = pdppa;
1661: pdppg0 = pdppg;
1.40 ryo 1662: pdppa = pmap_alloc_pdp(pm, &pdppg, flags, false); /* L2 pdp */
1.36 ryo 1663: if (pdppa == POOL_PADDR_INVALID) {
1664: if (flags & PMAP_CANFAIL) {
1665: error = ENOMEM;
1.39 ryo 1666: goto fail0;
1.36 ryo 1667: }
1.39 ryo 1668: pm_unlock(pm);
1.36 ryo 1669: panic("%s: cannot allocate L2 table", __func__);
1670: }
1.2 ryo 1671: atomic_swap_64(&l1[idx], pdppa | L1_TABLE);
1.36 ryo 1672: _pmap_pdp_addref(pm, pdppa0, pdppg0); /* L1 occupancy++ */
1673: _pmap_pdp_setparent(pm, pdppg, &l1[idx]);
1.19 ryo 1674: l3only = false;
1.2 ryo 1675: } else {
1676: pdppa = l1pde_pa(pde);
1.36 ryo 1677: pdppg = NULL;
1.2 ryo 1678: }
1.9 christos 1679: l2 = (void *)AARCH64_PA_TO_KVA(pdppa);
1.2 ryo 1680:
1681: idx = l2pde_index(va);
1682: pde = l2[idx];
1683: if (!l2pde_valid(pde)) {
1.36 ryo 1684: pdppa0 = pdppa;
1685: pdppg0 = pdppg;
1.40 ryo 1686: pdppa = pmap_alloc_pdp(pm, &pdppg, flags, false); /* L3 pdp */
1.36 ryo 1687: if (pdppa == POOL_PADDR_INVALID) {
1688: if (flags & PMAP_CANFAIL) {
1689: error = ENOMEM;
1.39 ryo 1690: goto fail0;
1.36 ryo 1691: }
1.39 ryo 1692: pm_unlock(pm);
1.36 ryo 1693: panic("%s: cannot allocate L3 table", __func__);
1694: }
1.2 ryo 1695: atomic_swap_64(&l2[idx], pdppa | L2_TABLE);
1.36 ryo 1696: _pmap_pdp_addref(pm, pdppa0, pdppg0); /* L2 occupancy++ */
1697: _pmap_pdp_setparent(pm, pdppg, &l2[idx]);
1.19 ryo 1698: l3only = false;
1.2 ryo 1699: } else {
1700: pdppa = l2pde_pa(pde);
1.36 ryo 1701: pdppg = NULL;
1.2 ryo 1702: }
1.9 christos 1703: l3 = (void *)AARCH64_PA_TO_KVA(pdppa);
1.2 ryo 1704:
1705: idx = l3pte_index(va);
1706: ptep = &l3[idx]; /* as PTE */
1707:
1.39 ryo 1708: opte = atomic_swap_64(ptep, 0);
1.32 ryo 1709: need_sync_icache = (prot & VM_PROT_EXECUTE);
1.2 ryo 1710:
1.40 ryo 1711: /* for lock ordering for pg and opg */
1712: pgs[0] = pg;
1713: pgs[1] = NULL;
1.39 ryo 1714:
1715: /* remap? */
1716: if (l3pte_valid(opte)) {
1717: struct vm_page *opg;
1718: bool need_remove_pv;
1719:
1.2 ryo 1720: KASSERT(!kenter); /* pmap_kenter_pa() cannot override */
1.39 ryo 1721: #ifdef PMAPCOUNTERS
1.2 ryo 1722: PMAP_COUNT(remappings);
1.39 ryo 1723: if (opte & LX_BLKPAG_OS_WIRED) {
1724: PMSTAT_DEC_WIRED_COUNT(pm);
1725: }
1726: PMSTAT_DEC_RESIDENT_COUNT(pm);
1727: if (user) {
1728: PMAP_COUNT(user_mappings_changed);
1729: } else {
1730: PMAP_COUNT(kern_mappings_changed);
1731: }
1732: #endif
1733: UVMHIST_LOG(pmaphist,
1734: "va=%016lx has already mapped."
1735: " old-pa=%016lx new-pa=%016lx, old-pte=%016llx\n",
1736: va, l3pte_pa(opte), pa, opte);
1737:
1738: if (pa == l3pte_pa(opte)) {
1739: /* old and new pte have same pa, no need to update pv */
1740: need_remove_pv = (pg == NULL);
1741: need_update_pv = false;
1742: if (need_sync_icache && l3pte_executable(opte, user))
1.32 ryo 1743: need_sync_icache = false;
1744: } else {
1.39 ryo 1745: need_remove_pv = true;
1746: }
1.40 ryo 1747:
1748: if (need_remove_pv &&
1749: ((opg = PHYS_TO_VM_PAGE(l3pte_pa(opte))) != NULL)) {
1750: /* need to lock both pg and opg against deadlock */
1751: if (pg < opg) {
1752: pgs[0] = pg;
1753: pgs[1] = opg;
1754: } else {
1755: pgs[0] = opg;
1756: pgs[1] = pg;
1.2 ryo 1757: }
1.40 ryo 1758: pmap_pv_lock(VM_PAGE_TO_MD(pgs[0]));
1759: pmap_pv_lock(VM_PAGE_TO_MD(pgs[1]));
1760: opv = _pmap_remove_pv(opg, pm, va, opte);
1761: } else {
1762: if (pg != NULL)
1763: pmap_pv_lock(VM_PAGE_TO_MD(pg));
1.36 ryo 1764: }
1.40 ryo 1765: } else {
1766: if (pg != NULL)
1767: pmap_pv_lock(VM_PAGE_TO_MD(pg));
1.2 ryo 1768: }
1769:
1.40 ryo 1770: if (!l3pte_valid(opte))
1771: _pmap_pdp_addref(pm, pdppa, pdppg); /* L3 occupancy++ */
1772:
1.20 ryo 1773: /*
1774: * read permission is treated as an access permission internally.
1775: * require to add PROT_READ even if only PROT_WRITE or PROT_EXEC
1776: */
1.44 ryo 1777: if (prot & (VM_PROT_WRITE|VM_PROT_EXECUTE))
1.20 ryo 1778: prot |= VM_PROT_READ;
1.46 ryo 1779: if (flags & (VM_PROT_WRITE|VM_PROT_EXECUTE))
1780: flags |= VM_PROT_READ;
1.20 ryo 1781:
1.2 ryo 1782: mdattr = VM_PROT_READ | VM_PROT_WRITE;
1.39 ryo 1783: if (need_update_pv) {
1.12 ryo 1784: error = _pmap_enter_pv(pg, pm, &spv, va, ptep, pa, flags);
1.3 ryo 1785: if (error != 0) {
1.11 ryo 1786: /*
1787: * If pmap_enter() fails,
1788: * it must not leave behind an existing pmap entry.
1789: */
1.39 ryo 1790: if (lxpde_valid(opte)) {
1791: bool pdpremoved = _pmap_pdp_delref(pm,
1792: AARCH64_KVA_TO_PA(trunc_page(
1793: (vaddr_t)ptep)), true);
1794: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid,
1795: va, !pdpremoved);
1796: }
1.3 ryo 1797: PMAP_COUNT(pv_entry_cannotalloc);
1798: if (flags & PMAP_CANFAIL)
1.39 ryo 1799: goto fail1;
1.11 ryo 1800: panic("pmap_enter: failed to allocate pv_entry");
1.3 ryo 1801: }
1.39 ryo 1802: }
1.2 ryo 1803:
1.39 ryo 1804: if (pg != NULL) {
1.7 ryo 1805: /* update referenced/modified flags */
1806: VM_PAGE_TO_MD(pg)->mdpg_flags |=
1.46 ryo 1807: (flags & (VM_PROT_READ | VM_PROT_WRITE));
1.7 ryo 1808: mdattr &= VM_PAGE_TO_MD(pg)->mdpg_flags;
1.2 ryo 1809: }
1810:
1811: #ifdef PMAPCOUNTERS
1812: switch (flags & PMAP_CACHE_MASK) {
1813: case PMAP_NOCACHE:
1814: case PMAP_NOCACHE_OVR:
1815: PMAP_COUNT(uncached_mappings);
1816: break;
1817: }
1818: #endif
1819:
1.18 ryo 1820: attr = _pmap_pte_adjust_prot(L3_PAGE, prot, mdattr, user);
1.2 ryo 1821: attr = _pmap_pte_adjust_cacheflags(attr, flags);
1822: if (VM_MAXUSER_ADDRESS > va)
1823: attr |= LX_BLKPAG_APUSER;
1.3 ryo 1824: if (flags & PMAP_WIRED)
1825: attr |= LX_BLKPAG_OS_WIRED;
1.2 ryo 1826: #ifdef MULTIPROCESSOR
1827: attr |= LX_BLKPAG_SH_IS;
1828: #endif
1829:
1830: pte = pa | attr;
1.13 ryo 1831:
1.32 ryo 1832: if (need_sync_icache) {
1.15 ryo 1833: /* non-exec -> exec */
1834: UVMHIST_LOG(pmaphist,
1835: "icache_sync: pm=%p, va=%016lx, pte: %016lx -> %016lx",
1.13 ryo 1836: pm, va, opte, pte);
1.15 ryo 1837: if (!l3pte_writable(pte)) {
1.19 ryo 1838: PTE_ICACHE_SYNC_PAGE(pte, ptep, pm, va, l3only);
1.15 ryo 1839: atomic_swap_64(ptep, pte);
1.19 ryo 1840: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va ,true);
1.15 ryo 1841: } else {
1842: atomic_swap_64(ptep, pte);
1.19 ryo 1843: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, l3only);
1.15 ryo 1844: cpu_icache_sync_range(va, PAGE_SIZE);
1845: }
1846: } else {
1847: atomic_swap_64(ptep, pte);
1.19 ryo 1848: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, l3only);
1.15 ryo 1849: }
1.39 ryo 1850:
1.36 ryo 1851: if (pte & LX_BLKPAG_OS_WIRED) {
1852: PMSTAT_INC_WIRED_COUNT(pm);
1853: }
1854: PMSTAT_INC_RESIDENT_COUNT(pm);
1.2 ryo 1855:
1.39 ryo 1856: fail1:
1.40 ryo 1857: if (pgs[1] != NULL)
1858: pmap_pv_unlock(VM_PAGE_TO_MD(pgs[1]));
1859: if (pgs[0] != NULL)
1860: pmap_pv_unlock(VM_PAGE_TO_MD(pgs[0]));
1.39 ryo 1861: fail0:
1.12 ryo 1862: pm_unlock(pm);
1863:
1864: /* spare pv was not used. discard */
1865: if (spv != NULL)
1866: pool_cache_put(&_pmap_pv_pool, spv);
1867:
1.14 ryo 1868: if (opv != NULL)
1869: pool_cache_put(&_pmap_pv_pool, opv);
1870:
1.2 ryo 1871: return error;
1872: }
1873:
1874: int
1875: pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1.1 matt 1876: {
1.2 ryo 1877: return _pmap_enter(pm, va, pa, prot, flags, false);
1.1 matt 1878: }
1879:
1880: void
1.2 ryo 1881: pmap_remove_all(struct pmap *pm)
1.1 matt 1882: {
1.2 ryo 1883: /* nothing to do */
1.1 matt 1884: }
1885:
1.2 ryo 1886: static void
1.35 ryo 1887: _pmap_remove(struct pmap *pm, vaddr_t sva, vaddr_t eva, bool kremove,
1888: struct pv_entry **pvtofree)
1.1 matt 1889: {
1.35 ryo 1890: pt_entry_t pte, *ptep = NULL;
1.2 ryo 1891: struct vm_page *pg;
1.35 ryo 1892: struct pv_entry *opv;
1.2 ryo 1893: paddr_t pa;
1.35 ryo 1894: vaddr_t va;
1895: vsize_t blocksize = 0;
1.36 ryo 1896: bool pdpremoved;
1.2 ryo 1897:
1898: UVMHIST_FUNC(__func__);
1899: UVMHIST_CALLED(pmaphist);
1900:
1.35 ryo 1901: UVMHIST_LOG(pmaphist, "pm=%p, sva=%016lx, eva=%016lx, kremovemode=%d",
1902: pm, sva, eva, kremove);
1.2 ryo 1903:
1.35 ryo 1904: for (va = sva; (va < eva) && (pm->pm_stats.resident_count != 0);
1905: va = (va + blocksize) & ~(blocksize - 1)) {
1906:
1907: /* va is belong to the same L3 table as before? */
1908: if ((blocksize == L3_SIZE) && ((va & L3INDEXMASK) != 0))
1909: ptep++;
1910: else
1911: ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
1.8 ryo 1912:
1.2 ryo 1913: pte = *ptep;
1.35 ryo 1914: if (!lxpde_valid(pte))
1915: continue;
1.2 ryo 1916:
1.35 ryo 1917: if (!kremove) {
1918: pa = lxpde_pa(pte);
1.3 ryo 1919: pg = PHYS_TO_VM_PAGE(pa);
1.35 ryo 1920: if (pg != NULL) {
1.39 ryo 1921:
1922: pmap_pv_lock(VM_PAGE_TO_MD(pg));
1.35 ryo 1923: opv = _pmap_remove_pv(pg, pm, va, pte);
1.39 ryo 1924: pmap_pv_unlock(VM_PAGE_TO_MD(pg));
1.35 ryo 1925: if (opv != NULL) {
1926: opv->pv_next = *pvtofree;
1927: *pvtofree = opv;
1928: }
1929: }
1930: }
1.2 ryo 1931:
1.36 ryo 1932: pte = atomic_swap_64(ptep, 0);
1933: if (!lxpde_valid(pte))
1934: continue;
1935:
1936: pdpremoved = _pmap_pdp_delref(pm,
1937: AARCH64_KVA_TO_PA(trunc_page((vaddr_t)ptep)), true);
1938: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, !pdpremoved);
1.3 ryo 1939:
1.36 ryo 1940: if (pdpremoved) {
1941: /*
1942: * this Ln page table page has been removed.
1943: * skip to next Ln table
1944: */
1945: blocksize *= Ln_ENTRIES;
1946: }
1947:
1948: if ((pte & LX_BLKPAG_OS_WIRED) != 0) {
1949: PMSTAT_DEC_WIRED_COUNT(pm);
1950: }
1951: PMSTAT_DEC_RESIDENT_COUNT(pm);
1.2 ryo 1952: }
1.1 matt 1953: }
1954:
1955: void
1.2 ryo 1956: pmap_remove(struct pmap *pm, vaddr_t sva, vaddr_t eva)
1.1 matt 1957: {
1.35 ryo 1958: struct pv_entry *pvtofree = NULL;
1959: struct pv_entry *pv, *pvtmp;
1.2 ryo 1960:
1.28 ryo 1961: KASSERT_PM_ADDR(pm, sva);
1.2 ryo 1962: KASSERT(!IN_KSEG_ADDR(sva));
1963:
1.35 ryo 1964: pm_lock(pm);
1965: _pmap_remove(pm, sva, eva, false, &pvtofree);
1966: pm_unlock(pm);
1967:
1968: for (pv = pvtofree; pv != NULL; pv = pvtmp) {
1969: pvtmp = pv->pv_next;
1970: pool_cache_put(&_pmap_pv_pool, pv);
1971: }
1.1 matt 1972: }
1973:
1974: void
1.2 ryo 1975: pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
1.1 matt 1976: {
1.2 ryo 1977: struct vm_page_md *md = VM_PAGE_TO_MD(pg);
1978: struct pv_entry *pv, *pvtmp;
1.3 ryo 1979: pt_entry_t opte;
1.2 ryo 1980:
1981: KASSERT((prot & VM_PROT_READ) || !(prot & VM_PROT_WRITE));
1982:
1983: UVMHIST_FUNC(__func__);
1984: UVMHIST_CALLED(pmaphist);
1985:
1986: UVMHIST_LOG(pmaphist, "pg=%p, md=%p, pa=%016lx, prot=%08x",
1987: pg, md, VM_PAGE_TO_PHYS(pg), prot);
1988:
1989:
1990: if ((prot & (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
1991: VM_PROT_NONE) {
1.35 ryo 1992: struct pv_entry *pvtofree = NULL;
1.2 ryo 1993:
1994: /* remove all pages reference to this physical page */
1995: pmap_pv_lock(md);
1996: TAILQ_FOREACH_SAFE(pv, &md->mdpg_pvhead, pv_link, pvtmp) {
1997:
1.3 ryo 1998: opte = atomic_swap_64(pv->pv_ptep, 0);
1.36 ryo 1999: if (lxpde_valid(opte)) {
2000: _pmap_pdp_delref(pv->pv_pmap,
2001: AARCH64_KVA_TO_PA(trunc_page(
2002: (vaddr_t)pv->pv_ptep)), false);
2003: AARCH64_TLBI_BY_ASID_VA(pv->pv_pmap->pm_asid,
2004: pv->pv_va, true);
1.2 ryo 2005:
1.36 ryo 2006: if ((opte & LX_BLKPAG_OS_WIRED) != 0) {
2007: PMSTAT_DEC_WIRED_COUNT(pv->pv_pmap);
2008: }
2009: PMSTAT_DEC_RESIDENT_COUNT(pv->pv_pmap);
2010: }
1.2 ryo 2011: TAILQ_REMOVE(&md->mdpg_pvhead, pv, pv_link);
2012: PMAP_COUNT(pv_remove);
1.35 ryo 2013:
2014: pv->pv_next = pvtofree;
2015: pvtofree = pv;
1.2 ryo 2016: }
2017: pmap_pv_unlock(md);
2018:
1.35 ryo 2019: for (pv = pvtofree; pv != NULL; pv = pvtmp) {
2020: pvtmp = pv->pv_next;
2021: pool_cache_put(&_pmap_pv_pool, pv);
2022: }
1.2 ryo 2023: } else {
2024: pmap_pv_lock(md);
2025: TAILQ_FOREACH(pv, &md->mdpg_pvhead, pv_link) {
1.3 ryo 2026: _pmap_protect_pv(pg, pv, prot);
1.2 ryo 2027: }
2028: pmap_pv_unlock(md);
2029: }
1.1 matt 2030: }
2031:
2032: void
1.2 ryo 2033: pmap_unwire(struct pmap *pm, vaddr_t va)
1.1 matt 2034: {
1.2 ryo 2035: pt_entry_t pte, *ptep;
2036:
2037: UVMHIST_FUNC(__func__);
2038: UVMHIST_CALLED(pmaphist);
2039:
2040: UVMHIST_LOG(pmaphist, "pm=%p, va=%016lx", pm, va, 0, 0);
2041:
2042: PMAP_COUNT(unwire);
2043:
1.28 ryo 2044: KASSERT_PM_ADDR(pm, va);
2045: KASSERT(!IN_KSEG_ADDR(va));
1.2 ryo 2046:
1.11 ryo 2047: pm_lock(pm);
1.27 ryo 2048: ptep = _pmap_pte_lookup_l3(pm, va);
1.2 ryo 2049: if (ptep != NULL) {
2050: pte = *ptep;
1.3 ryo 2051: if (!l3pte_valid(pte) ||
2052: ((pte & LX_BLKPAG_OS_WIRED) == 0)) {
2053: /* invalid pte, or pte is not wired */
2054: PMAP_COUNT(unwire_failure);
1.11 ryo 2055: pm_unlock(pm);
1.2 ryo 2056: return;
2057: }
2058:
1.3 ryo 2059: pte &= ~LX_BLKPAG_OS_WIRED;
2060: atomic_swap_64(ptep, pte);
1.2 ryo 2061:
1.36 ryo 2062: PMSTAT_DEC_WIRED_COUNT(pm);
1.2 ryo 2063: }
1.11 ryo 2064: pm_unlock(pm);
1.1 matt 2065: }
2066:
1.2 ryo 2067: bool
2068: pmap_fault_fixup(struct pmap *pm, vaddr_t va, vm_prot_t accessprot, bool user)
1.1 matt 2069: {
1.2 ryo 2070: struct vm_page *pg;
2071: struct vm_page_md *md;
2072: pt_entry_t *ptep, pte;
2073: vm_prot_t pmap_prot;
2074: paddr_t pa;
1.8 ryo 2075: bool fixed = false;
1.2 ryo 2076:
2077: UVMHIST_FUNC(__func__);
2078: UVMHIST_CALLED(pmaphist);
2079:
2080: UVMHIST_LOG(pmaphist, "pm=%p, va=%016lx, accessprot=%08x",
2081: pm, va, accessprot, 0);
2082:
2083:
2084: #if 0
1.28 ryo 2085: KASSERT_PM_ADDR(pm, va);
1.2 ryo 2086: #else
2087: if (((pm == pmap_kernel()) &&
1.28 ryo 2088: !(IN_RANGE(va, VM_MIN_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS))) ||
1.2 ryo 2089: ((pm != pmap_kernel()) &&
1.28 ryo 2090: !(IN_RANGE(va, VM_MIN_ADDRESS, VM_MAX_ADDRESS)))) {
1.2 ryo 2091:
2092: UVMHIST_LOG(pmaphist,
2093: "pmap space and va mismatch: pm=%s, va=%016lx",
2094: (pm == pmap_kernel()) ? "kernel" : "user", va, 0, 0);
2095: return false;
2096: }
2097: #endif
2098:
1.11 ryo 2099: pm_lock(pm);
1.8 ryo 2100:
1.27 ryo 2101: ptep = _pmap_pte_lookup_l3(pm, va);
1.2 ryo 2102: if (ptep == NULL) {
2103: UVMHIST_LOG(pmaphist, "pte_lookup failure: va=%016lx",
2104: va, 0, 0, 0);
1.8 ryo 2105: goto done;
1.2 ryo 2106: }
2107:
2108: pte = *ptep;
2109: if (!l3pte_valid(pte)) {
2110: UVMHIST_LOG(pmaphist, "invalid pte: %016llx: va=%016lx",
2111: pte, va, 0, 0);
1.8 ryo 2112: goto done;
1.2 ryo 2113: }
2114:
2115: pa = l3pte_pa(*ptep);
2116: pg = PHYS_TO_VM_PAGE(pa);
2117: if (pg == NULL) {
2118: UVMHIST_LOG(pmaphist, "pg not found: va=%016lx", va, 0, 0, 0);
1.8 ryo 2119: goto done;
1.2 ryo 2120: }
2121: md = VM_PAGE_TO_MD(pg);
2122:
2123: /* get prot by pmap_enter() (stored in software use bit in pte) */
2124: switch (pte & (LX_BLKPAG_OS_READ|LX_BLKPAG_OS_WRITE)) {
2125: case 0:
2126: default:
2127: pmap_prot = 0;
2128: break;
2129: case LX_BLKPAG_OS_READ:
2130: pmap_prot = VM_PROT_READ;
2131: break;
2132: case LX_BLKPAG_OS_WRITE:
2133: case LX_BLKPAG_OS_READ|LX_BLKPAG_OS_WRITE:
2134: pmap_prot = (VM_PROT_READ|VM_PROT_WRITE);
2135: break;
2136: }
1.18 ryo 2137: if (l3pte_executable(pte, pm != pmap_kernel()))
1.2 ryo 2138: pmap_prot |= VM_PROT_EXECUTE;
2139:
2140: UVMHIST_LOG(pmaphist, "va=%016lx, pmapprot=%08x, accessprot=%08x",
2141: va, pmap_prot, accessprot, 0);
2142:
2143: /* ignore except read/write */
2144: accessprot &= (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE);
2145:
1.61 ! ryo 2146: /* PROT_EXEC requires implicit PROT_READ */
! 2147: if (accessprot & VM_PROT_EXECUTE)
! 2148: accessprot |= VM_PROT_READ;
! 2149:
1.2 ryo 2150: /* no permission to read/write/execute for this page */
2151: if ((pmap_prot & accessprot) != accessprot) {
2152: UVMHIST_LOG(pmaphist, "no permission to access", 0, 0, 0, 0);
1.8 ryo 2153: goto done;
1.2 ryo 2154: }
2155:
1.24 ryo 2156: /* pte is readable and writable, but occured fault? probably copy(9) */
2157: if ((pte & LX_BLKPAG_AF) && ((pte & LX_BLKPAG_AP) == LX_BLKPAG_AP_RW))
1.8 ryo 2158: goto done;
1.2 ryo 2159:
2160: pmap_pv_lock(md);
2161: if ((pte & LX_BLKPAG_AF) == 0) {
2162: /* pte has no AF bit, set referenced and AF bit */
2163: UVMHIST_LOG(pmaphist,
2164: "REFERENCED:"
2165: " va=%016lx, pa=%016lx, pte_prot=%08x, accessprot=%08x",
2166: va, pa, pmap_prot, accessprot);
2167: md->mdpg_flags |= VM_PROT_READ; /* set referenced */
2168: pte |= LX_BLKPAG_AF;
2169:
2170: PMAP_COUNT(fixup_referenced);
2171: }
2172: if ((accessprot & VM_PROT_WRITE) &&
2173: ((pte & LX_BLKPAG_AP) == LX_BLKPAG_AP_RO)) {
2174: /* pte is not RW. set modified and RW */
2175:
2176: UVMHIST_LOG(pmaphist, "MODIFIED:"
2177: " va=%016lx, pa=%016lx, pte_prot=%08x, accessprot=%08x",
2178: va, pa, pmap_prot, accessprot);
2179: md->mdpg_flags |= VM_PROT_WRITE; /* set modified */
2180: pte &= ~LX_BLKPAG_AP;
2181: pte |= LX_BLKPAG_AP_RW;
2182:
2183: PMAP_COUNT(fixup_modified);
2184: }
2185: pmap_pv_unlock(md);
2186:
2187: atomic_swap_64(ptep, pte);
1.19 ryo 2188: AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
2189:
1.8 ryo 2190: fixed = true;
1.2 ryo 2191:
1.8 ryo 2192: done:
1.11 ryo 2193: pm_unlock(pm);
1.8 ryo 2194: return fixed;
1.1 matt 2195: }
2196:
2197: bool
2198: pmap_clear_modify(struct vm_page *pg)
2199: {
1.2 ryo 2200: struct pv_entry *pv;
2201: struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
2202: pt_entry_t *ptep, pte, opte;
2203: vaddr_t va;
2204:
2205: UVMHIST_FUNC(__func__);
2206: UVMHIST_CALLED(pmaphist);
2207:
2208: UVMHIST_LOG(pmaphist, "pg=%p, mdpg_flags=%08x",
2209: pg, md->mdpg_flags, 0, 0);
2210:
2211: pmap_pv_lock(md);
2212:
2213: if ((md->mdpg_flags & VM_PROT_WRITE) == 0) {
2214: pmap_pv_unlock(md);
2215: return false;
2216: }
2217:
2218: md->mdpg_flags &= ~VM_PROT_WRITE;
2219:
2220: PMAP_COUNT(clear_modify);
2221: TAILQ_FOREACH(pv, &md->mdpg_pvhead, pv_link) {
2222: PMAP_COUNT(clear_modify_pages);
2223:
2224: va = pv->pv_va;
2225:
2226: ptep = pv->pv_ptep;
2227: opte = pte = *ptep;
2228: tryagain:
2229: if (!l3pte_valid(pte))
2230: continue;
2231:
2232: /* clear write permission */
2233: pte &= ~LX_BLKPAG_AP;
2234: pte |= LX_BLKPAG_AP_RO;
2235:
2236: /* XXX: possible deadlock if using PM_LOCK(). this is racy */
2237: if ((pte = atomic_cas_64(ptep, opte, pte)) != opte) {
2238: opte = pte;
2239: goto tryagain;
2240: }
2241:
1.19 ryo 2242: AARCH64_TLBI_BY_ASID_VA(pv->pv_pmap->pm_asid, va, true);
1.2 ryo 2243:
2244: UVMHIST_LOG(pmaphist,
2245: "va=%016llx, ptep=%p, pa=%016lx, RW -> RO",
2246: va, ptep, l3pte_pa(pte), 0);
2247: }
2248:
2249: pmap_pv_unlock(md);
2250:
2251: return true;
1.1 matt 2252: }
2253:
2254: bool
2255: pmap_clear_reference(struct vm_page *pg)
2256: {
1.2 ryo 2257: struct pv_entry *pv;
2258: struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
2259: pt_entry_t *ptep, pte, opte;
2260: vaddr_t va;
2261:
2262: UVMHIST_FUNC(__func__);
2263: UVMHIST_CALLED(pmaphist);
2264:
2265: UVMHIST_LOG(pmaphist, "pg=%p, mdpg_flags=%08x",
2266: pg, md->mdpg_flags, 0, 0);
2267:
2268: pmap_pv_lock(md);
2269:
2270: if ((md->mdpg_flags & VM_PROT_READ) == 0) {
2271: pmap_pv_unlock(md);
2272: return false;
2273: }
2274: md->mdpg_flags &= ~VM_PROT_READ;
2275:
2276: PMAP_COUNT(clear_reference);
2277: TAILQ_FOREACH(pv, &md->mdpg_pvhead, pv_link) {
2278: PMAP_COUNT(clear_reference_pages);
2279:
2280: va = pv->pv_va;
2281:
2282: ptep = pv->pv_ptep;
2283: opte = pte = *ptep;
2284: tryagain:
2285: if (!l3pte_valid(pte))
2286: continue;
2287:
2288: /* clear access permission */
2289: pte &= ~LX_BLKPAG_AF;
2290:
2291: /* XXX: possible deadlock if using PM_LOCK(). this is racy */
2292: if ((pte = atomic_cas_64(ptep, opte, pte)) != opte) {
2293: opte = pte;
2294: goto tryagain;
2295: }
2296:
1.19 ryo 2297: AARCH64_TLBI_BY_ASID_VA(pv->pv_pmap->pm_asid, va, true);
1.2 ryo 2298:
2299: UVMHIST_LOG(pmaphist, "va=%016llx, ptep=%p, pa=%016lx, unse AF",
2300: va, ptep, l3pte_pa(pte), 0);
2301: }
2302:
2303: pmap_pv_unlock(md);
2304:
2305: return true;
1.1 matt 2306: }
2307:
2308: bool
2309: pmap_is_modified(struct vm_page *pg)
2310: {
1.2 ryo 2311: struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
1.3 ryo 2312:
1.2 ryo 2313: return (md->mdpg_flags & VM_PROT_WRITE);
1.1 matt 2314: }
2315:
2316: bool
2317: pmap_is_referenced(struct vm_page *pg)
2318: {
1.3 ryo 2319: struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
2320:
2321: return (md->mdpg_flags & VM_PROT_READ);
1.2 ryo 2322: }
2323:
2324: #ifdef DDB
1.37 ryo 2325:
1.17 ryo 2326: /* get pointer to kernel segment L2 or L3 table entry */
2327: pt_entry_t *
2328: kvtopte(vaddr_t va)
2329: {
1.28 ryo 2330: KASSERT(IN_RANGE(va, VM_MIN_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS));
1.17 ryo 2331:
1.27 ryo 2332: return _pmap_pte_lookup_bs(pmap_kernel(), va, NULL);
1.17 ryo 2333: }
2334:
2335: /* change attribute of kernel segment */
2336: pt_entry_t
2337: pmap_kvattr(vaddr_t va, vm_prot_t prot)
2338: {
2339: pt_entry_t *ptep, pte, opte;
2340:
1.28 ryo 2341: KASSERT(IN_RANGE(va, VM_MIN_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS));
1.17 ryo 2342:
2343: ptep = kvtopte(va);
2344: if (ptep == NULL)
2345: panic("%s: %016lx is not mapped\n", __func__, va);
2346:
2347: opte = pte = *ptep;
2348:
2349: pte &= ~(LX_BLKPAG_AF|LX_BLKPAG_AP);
2350: switch (prot & (VM_PROT_READ|VM_PROT_WRITE)) {
2351: case 0:
2352: break;
2353: case VM_PROT_READ:
2354: pte |= (LX_BLKPAG_AF|LX_BLKPAG_AP_RO);
2355: break;
2356: case VM_PROT_WRITE:
2357: case VM_PROT_READ|VM_PROT_WRITE:
2358: pte |= (LX_BLKPAG_AF|LX_BLKPAG_AP_RW);
2359: break;
2360: }
2361:
2362: if ((prot & VM_PROT_EXECUTE) == 0) {
1.18 ryo 2363: pte |= LX_BLKPAG_PXN;
1.17 ryo 2364: } else {
2365: pte |= LX_BLKPAG_AF;
1.18 ryo 2366: pte &= ~LX_BLKPAG_PXN;
1.17 ryo 2367: }
2368:
2369: *ptep = pte;
2370:
2371: return opte;
2372: }
2373:
1.26 ryo 2374: void
2375: pmap_db_pte_print(pt_entry_t pte, int level,
2376: void (*pr)(const char *, ...) __printflike(1, 2))
1.2 ryo 2377: {
2378: if (pte == 0) {
1.8 ryo 2379: pr(" UNUSED\n");
1.26 ryo 2380: return;
2381: }
2382:
2383: pr(" %s", (pte & LX_VALID) ? "VALID" : "**INVALID**");
2384:
2385: if ((level == 0) ||
2386: ((level == 1) && l1pde_is_table(pte)) ||
2387: ((level == 2) && l2pde_is_table(pte))) {
2388:
2389: /* L0/L1/L2 TABLE */
2390: if ((level == 0) && ((pte & LX_TYPE) != LX_TYPE_TBL))
2391: pr(" **ILLEGAL TYPE**"); /* L0 doesn't support block */
2392: else
1.37 ryo 2393: pr(" L%d-TABLE", level);
1.2 ryo 2394:
1.26 ryo 2395: pr(", PA=%lx", l0pde_pa(pte));
1.2 ryo 2396:
1.26 ryo 2397: if (pte & LX_TBL_NSTABLE)
2398: pr(", NSTABLE");
2399: if (pte & LX_TBL_APTABLE)
2400: pr(", APTABLE");
2401: if (pte & LX_TBL_UXNTABLE)
2402: pr(", UXNTABLE");
2403: if (pte & LX_TBL_PXNTABLE)
2404: pr(", PXNTABLE");
1.2 ryo 2405:
2406: } else if (((level == 1) && l1pde_is_block(pte)) ||
2407: ((level == 2) && l2pde_is_block(pte)) ||
1.26 ryo 2408: (level == 3)) {
1.2 ryo 2409:
1.26 ryo 2410: /* L1/L2 BLOCK or L3 PAGE */
1.37 ryo 2411: switch (level) {
2412: case 1:
2413: pr(" L1(1G)-BLOCK");
2414: break;
2415: case 2:
2416: pr(" L2(2M)-BLOCK");
2417: break;
2418: case 3:
1.26 ryo 2419: pr(" %s", l3pte_is_page(pte) ?
1.37 ryo 2420: "L3(4K)-PAGE" : "**ILLEGAL TYPE**");
2421: break;
2422: }
1.2 ryo 2423:
1.26 ryo 2424: pr(", PA=%lx", l3pte_pa(pte));
1.2 ryo 2425:
1.37 ryo 2426: pr(", %s", (pte & LX_BLKPAG_UXN) ?
1.43 skrll 2427: "UXN" : "UX ");
1.37 ryo 2428: pr(", %s", (pte & LX_BLKPAG_PXN) ?
1.43 skrll 2429: "PXN" : "PX ");
1.2 ryo 2430:
2431: if (pte & LX_BLKPAG_CONTIG)
1.26 ryo 2432: pr(", CONTIG");
1.2 ryo 2433:
1.26 ryo 2434: pr(", %s", (pte & LX_BLKPAG_NG) ? "NG" : "global");
1.37 ryo 2435: pr(", %s", (pte & LX_BLKPAG_AF) ?
2436: "accessible" :
2437: "**fault** ");
1.2 ryo 2438:
2439: switch (pte & LX_BLKPAG_SH) {
2440: case LX_BLKPAG_SH_NS:
2441: pr(", SH_NS");
2442: break;
2443: case LX_BLKPAG_SH_OS:
2444: pr(", SH_OS");
2445: break;
2446: case LX_BLKPAG_SH_IS:
2447: pr(", SH_IS");
2448: break;
2449: default:
2450: pr(", SH_??");
2451: break;
2452: }
2453:
2454: pr(", %s", (pte & LX_BLKPAG_AP_RO) ? "RO" : "RW");
2455: pr(", %s", (pte & LX_BLKPAG_APUSER) ? "EL0" : "EL1");
1.26 ryo 2456: pr(", %s", (pte & LX_BLKPAG_NS) ? "NS" : "secure");
1.2 ryo 2457:
2458: switch (pte & LX_BLKPAG_ATTR_MASK) {
2459: case LX_BLKPAG_ATTR_NORMAL_WB:
1.43 skrll 2460: pr(", WB");
1.2 ryo 2461: break;
2462: case LX_BLKPAG_ATTR_NORMAL_NC:
1.43 skrll 2463: pr(", NC");
1.2 ryo 2464: break;
2465: case LX_BLKPAG_ATTR_NORMAL_WT:
1.43 skrll 2466: pr(", WT");
1.2 ryo 2467: break;
2468: case LX_BLKPAG_ATTR_DEVICE_MEM:
2469: pr(", DEVICE");
2470: break;
2471: }
2472:
1.26 ryo 2473: if (pte & LX_BLKPAG_OS_BOOT)
2474: pr(", boot");
1.2 ryo 2475: if (pte & LX_BLKPAG_OS_READ)
2476: pr(", pmap_read");
2477: if (pte & LX_BLKPAG_OS_WRITE)
2478: pr(", pmap_write");
1.26 ryo 2479: if (pte & LX_BLKPAG_OS_WIRED)
1.43 skrll 2480: pr(", wired");
1.2 ryo 2481: } else {
1.26 ryo 2482: pr(" **ILLEGAL TYPE**");
1.2 ryo 2483: }
2484: pr("\n");
1.1 matt 2485: }
2486:
1.2 ryo 2487: void
1.38 ryo 2488: pmap_db_pteinfo(vaddr_t va, void (*pr)(const char *, ...) __printflike(1, 2))
1.1 matt 2489: {
1.2 ryo 2490: struct vm_page *pg;
2491: bool user;
2492: pd_entry_t *l0, *l1, *l2, *l3;
2493: pd_entry_t pde;
2494: pt_entry_t pte;
2495: struct vm_page_md *md;
1.26 ryo 2496: uint64_t ttbr;
1.2 ryo 2497: paddr_t pa;
2498: unsigned int idx;
2499:
1.26 ryo 2500: if (va & TTBR_SEL_VA) {
2501: user = false;
2502: ttbr = reg_ttbr1_el1_read();
2503: } else {
2504: user = true;
2505: ttbr = reg_ttbr0_el1_read();
2506: }
2507: pa = ttbr & TTBR_BADDR;
2508: l0 = (pd_entry_t *)AARCH64_PA_TO_KVA(pa);
2509:
1.2 ryo 2510: /*
2511: * traverse L0 -> L1 -> L2 -> L3 table
2512: */
1.38 ryo 2513: pr("TTBR%d=%016"PRIx64", pa=%016"PRIxPADDR", va=%p",
2514: user ? 0 : 1, ttbr, pa, l0);
2515: pr(", input-va=%016"PRIxVADDR
2516: ", L0-index=%ld, L1-index=%ld, L2-index=%ld, L3-index=%ld\n",
1.2 ryo 2517: va,
2518: (va & L0_ADDR_BITS) >> L0_SHIFT,
2519: (va & L1_ADDR_BITS) >> L1_SHIFT,
2520: (va & L2_ADDR_BITS) >> L2_SHIFT,
2521: (va & L3_ADDR_BITS) >> L3_SHIFT);
2522:
2523: idx = l0pde_index(va);
2524: pde = l0[idx];
2525:
1.38 ryo 2526: pr("L0[%3d]=%016"PRIx64":", idx, pde);
1.2 ryo 2527: pmap_db_pte_print(pde, 0, pr);
2528:
2529: if (!l0pde_valid(pde))
2530: return;
2531:
1.26 ryo 2532: l1 = (pd_entry_t *)AARCH64_PA_TO_KVA(l0pde_pa(pde));
1.2 ryo 2533: idx = l1pde_index(va);
2534: pde = l1[idx];
2535:
1.38 ryo 2536: pr(" L1[%3d]=%016"PRIx64":", idx, pde);
1.2 ryo 2537: pmap_db_pte_print(pde, 1, pr);
2538:
2539: if (!l1pde_valid(pde) || l1pde_is_block(pde))
2540: return;
2541:
1.26 ryo 2542: l2 = (pd_entry_t *)AARCH64_PA_TO_KVA(l1pde_pa(pde));
1.2 ryo 2543: idx = l2pde_index(va);
2544: pde = l2[idx];
2545:
1.38 ryo 2546: pr(" L2[%3d]=%016"PRIx64":", idx, pde);
1.2 ryo 2547: pmap_db_pte_print(pde, 2, pr);
2548:
2549: if (!l2pde_valid(pde) || l2pde_is_block(pde))
2550: return;
2551:
1.26 ryo 2552: l3 = (pd_entry_t *)AARCH64_PA_TO_KVA(l2pde_pa(pde));
1.2 ryo 2553: idx = l3pte_index(va);
2554: pte = l3[idx];
2555:
1.38 ryo 2556: pr(" L3[%3d]=%016"PRIx64":", idx, pte);
1.2 ryo 2557: pmap_db_pte_print(pte, 3, pr);
2558:
2559: pa = l3pte_pa(pte);
2560: pg = PHYS_TO_VM_PAGE(pa);
1.8 ryo 2561: if (pg == NULL) {
2562: pr("No VM_PAGE\n");
2563: } else {
1.2 ryo 2564: pg_dump(pg, pr);
2565: md = VM_PAGE_TO_MD(pg);
2566: pv_dump(md, pr);
2567: }
1.1 matt 2568: }
1.37 ryo 2569:
2570: static void
2571: dump_ln_table(bool countmode, pd_entry_t *pdp, int level, int lnindex,
1.38 ryo 2572: vaddr_t va, void (*pr)(const char *, ...) __printflike(1, 2))
1.37 ryo 2573: {
2574: struct vm_page *pg;
2575: struct vm_page_md *md;
2576: pd_entry_t pde;
2577: paddr_t pa;
2578: int i, n;
2579: const char *spaces[4] = { " ", " ", " ", " " };
2580: const char *spc = spaces[level];
2581:
2582: pa = AARCH64_KVA_TO_PA((vaddr_t)pdp);
2583: pg = PHYS_TO_VM_PAGE(pa);
2584: md = VM_PAGE_TO_MD(pg);
2585:
2586: if (pg == NULL) {
2587: pr("%sL%d: pa=%lx pg=NULL\n", spc, level, pa);
2588: } else {
2589: pr("%sL%d: pa=%lx pg=%p, wire_count=%d, mdpg_ptep_parent=%p\n",
2590: spc, level, pa, pg, pg->wire_count, md->mdpg_ptep_parent);
2591: }
2592:
2593: for (i = n = 0; i < Ln_ENTRIES; i++) {
2594: db_read_bytes((db_addr_t)&pdp[i], sizeof(pdp[i]), (char *)&pde);
2595: if (lxpde_valid(pde)) {
2596: if (!countmode)
2597: pr("%sL%d[%3d] %3dth, va=%016lx, pte=%016lx:",
2598: spc, level, i, n, va, pde);
2599: n++;
2600:
2601: if (((level != 0) && (level != 3) &&
2602: l1pde_is_block(pde)) ||
2603: ((level == 3) && l3pte_is_page(pde))) {
2604: if (!countmode)
2605: pmap_db_pte_print(pde, level, pr);
2606: } else if ((level != 3) && l1pde_is_table(pde)) {
2607: if (!countmode)
2608: pmap_db_pte_print(pde, level, pr);
2609: pa = l0pde_pa(pde);
2610: dump_ln_table(countmode,
2611: (pd_entry_t *)AARCH64_PA_TO_KVA(pa),
2612: level + 1, i, va, pr);
2613: } else {
2614: if (!countmode)
2615: pmap_db_pte_print(pde, level, pr);
2616: }
2617: }
2618:
2619: switch (level) {
2620: case 0:
2621: va += L0_SIZE;
2622: break;
2623: case 1:
2624: va += L1_SIZE;
2625: break;
2626: case 2:
2627: va += L2_SIZE;
2628: break;
2629: case 3:
2630: va += L3_SIZE;
2631: break;
2632: }
2633: }
2634:
2635: if (level == 0)
2636: pr("L0 has %d entries\n", n);
2637: else
2638: pr("%sL%d[%3d] has %d L%d entries\n", spaces[level - 1],
2639: level - 1, lnindex, n, level);
2640:
2641: }
2642:
2643: static void
2644: pmap_db_dump_l0_table(bool countmode, pd_entry_t *pdp, vaddr_t va_base,
1.38 ryo 2645: void (*pr)(const char *, ...) __printflike(1, 2))
1.37 ryo 2646: {
2647: dump_ln_table(countmode, pdp, 0, 0, va_base, pr);
2648: }
2649:
2650: void
1.38 ryo 2651: pmap_db_ttbrdump(bool countmode, vaddr_t va,
2652: void (*pr)(const char *, ...) __printflike(1, 2))
1.37 ryo 2653: {
2654: struct pmap *pm, _pm;
2655:
2656: pm = (struct pmap *)va;
2657: db_read_bytes((db_addr_t)va, sizeof(_pm), (char *)&_pm);
2658:
2659: pr("pmap=%p\n", pm);
2660: pr(" pm_asid = %d\n", _pm.pm_asid);
2661: pr(" pm_l0table = %p\n", _pm.pm_l0table);
2662: pr(" pm_l0table_pa = %lx\n", _pm.pm_l0table_pa);
2663: pr(" pm_activated = %d\n\n", _pm.pm_activated);
2664:
2665: pmap_db_dump_l0_table(countmode, _pm.pm_l0table,
2666: (pm == pmap_kernel()) ? 0xffff000000000000UL : 0, pr);
2667: }
2668:
1.2 ryo 2669: #endif /* DDB */
CVSweb <webmaster@jp.NetBSD.org>