Annotation of src/sys/uvm/pmap/pmap_tlb.c, Revision 1.43
1.43 ! skrll 1: /* $NetBSD: pmap_tlb.c,v 1.42 2021/05/01 06:56:41 skrll Exp $ */
1.1 christos 2:
3: /*-
4: * Copyright (c) 2010 The NetBSD Foundation, Inc.
5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
8: * by Matt Thomas at 3am Software Foundry.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: *
19: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29: * POSSIBILITY OF SUCH DAMAGE.
30: */
31:
32: #include <sys/cdefs.h>
33:
1.43 ! skrll 34: __KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.42 2021/05/01 06:56:41 skrll Exp $");
1.1 christos 35:
36: /*
37: * Manages address spaces in a TLB.
38: *
39: * Normally there is a 1:1 mapping between a TLB and a CPU. However, some
40: * implementations may share a TLB between multiple CPUs (really CPU thread
41: * contexts). This requires the TLB abstraction to be separated from the
42: * CPU abstraction. It also requires that the TLB be locked while doing
43: * TLB activities.
44: *
45: * For each TLB, we track the ASIDs in use in a bitmap and a list of pmaps
46: * that have a valid ASID.
47: *
48: * We allocate ASIDs in increasing order until we have exhausted the supply,
49: * then reinitialize the ASID space, and start allocating again at 1. When
50: * allocating from the ASID bitmap, we skip any ASID who has a corresponding
51: * bit set in the ASID bitmap. Eventually this causes the ASID bitmap to fill
52: * and, when completely filled, a reinitialization of the ASID space.
53: *
54: * To reinitialize the ASID space, the ASID bitmap is reset and then the ASIDs
55: * of non-kernel TLB entries get recorded in the ASID bitmap. If the entries
56: * in TLB consume more than half of the ASID space, all ASIDs are invalidated,
57: * the ASID bitmap is recleared, and the list of pmaps is emptied. Otherwise,
58: * (the normal case), any ASID present in the TLB (even those which are no
59: * longer used by a pmap) will remain active (allocated) and all other ASIDs
60: * will be freed. If the size of the TLB is much smaller than the ASID space,
61: * this algorithm completely avoids TLB invalidation.
62: *
63: * For multiprocessors, we also have to deal TLB invalidation requests from
64: * other CPUs, some of which are dealt with the reinitialization of the ASID
65: * space. Whereas above we keep the ASIDs of those pmaps which have active
66: * TLB entries, this type of reinitialization preserves the ASIDs of any
67: * "onproc" user pmap and all other ASIDs will be freed. We must do this
68: * since we can't change the current ASID.
69: *
70: * Each pmap has two bitmaps: pm_active and pm_onproc. Each bit in pm_active
71: * indicates whether that pmap has an allocated ASID for a CPU. Each bit in
1.31 skrll 72: * pm_onproc indicates that the pmap's ASID is in use, i.e. a CPU has it in its
73: * "current ASID" field, e.g. the ASID field of the COP 0 register EntryHi for
74: * MIPS, or the ASID field of TTBR0 for AA64. The bit number used in these
75: * bitmaps comes from the CPU's cpu_index(). Even though these bitmaps contain
76: * the bits for all CPUs, the bits that correspond to the bits belonging to
77: * the CPUs sharing a TLB can only be manipulated while holding that TLB's
78: * lock. Atomic ops must be used to update them since multiple CPUs may be
79: * changing different sets of bits at same time but these sets never overlap.
1.1 christos 80: *
81: * When a change to the local TLB may require a change in the TLB's of other
82: * CPUs, we try to avoid sending an IPI if at all possible. For instance, if
83: * we are updating a PTE and that PTE previously was invalid and therefore
84: * couldn't support an active mapping, there's no need for an IPI since there
85: * can't be a TLB entry to invalidate. The other case is when we change a PTE
86: * to be modified we just update the local TLB. If another TLB has a stale
87: * entry, a TLB MOD exception will be raised and that will cause the local TLB
88: * to be updated.
89: *
90: * We never need to update a non-local TLB if the pmap doesn't have a valid
91: * ASID for that TLB. If it does have a valid ASID but isn't current "onproc"
92: * we simply reset its ASID for that TLB and then when it goes "onproc" it
93: * will allocate a new ASID and any existing TLB entries will be orphaned.
94: * Only in the case that pmap has an "onproc" ASID do we actually have to send
95: * an IPI.
96: *
97: * Once we determined we must send an IPI to shootdown a TLB, we need to send
98: * it to one of CPUs that share that TLB. We choose the lowest numbered CPU
99: * that has one of the pmap's ASID "onproc". In reality, any CPU sharing that
100: * TLB would do, but interrupting an active CPU seems best.
101: *
102: * A TLB might have multiple shootdowns active concurrently. The shootdown
103: * logic compresses these into a few cases:
104: * 0) nobody needs to have its TLB entries invalidated
105: * 1) one ASID needs to have its TLB entries invalidated
106: * 2) more than one ASID needs to have its TLB entries invalidated
107: * 3) the kernel needs to have its TLB entries invalidated
108: * 4) the kernel and one or more ASID need their TLB entries invalidated.
109: *
110: * And for each case we do:
111: * 0) nothing,
112: * 1) if that ASID is still "onproc", we invalidate the TLB entries for
113: * that single ASID. If not, just reset the pmap's ASID to invalidate
114: * and let it allocate a new ASID the next time it goes "onproc",
115: * 2) we reinitialize the ASID space (preserving any "onproc" ASIDs) and
116: * invalidate all non-wired non-global TLB entries,
117: * 3) we invalidate all of the non-wired global TLB entries,
118: * 4) we reinitialize the ASID space (again preserving any "onproc" ASIDs)
119: * invalidate all non-wired TLB entries.
120: *
121: * As you can see, shootdowns are not concerned with addresses, just address
122: * spaces. Since the number of TLB entries is usually quite small, this avoids
123: * a lot of overhead for not much gain.
124: */
125:
126: #define __PMAP_PRIVATE
127:
128: #include "opt_multiprocessor.h"
129:
130: #include <sys/param.h>
1.30 skrll 131:
1.1 christos 132: #include <sys/atomic.h>
1.30 skrll 133: #include <sys/cpu.h>
1.1 christos 134: #include <sys/kernel.h> /* for cold */
1.30 skrll 135: #include <sys/mutex.h>
136: #include <sys/proc.h>
137: #include <sys/systm.h>
1.1 christos 138:
139: #include <uvm/uvm.h>
140:
1.5 matt 141: static kmutex_t pmap_tlb0_lock __cacheline_aligned;
1.1 christos 142:
143: #define IFCONSTANT(x) (__builtin_constant_p((x)) ? (x) : 0)
144:
1.25 jdolecek 145: #if KERNEL_PID > 31
146: #error "KERNEL_PID expected in range 0-31"
147: #endif
148:
1.26 jdolecek 149: #define TLBINFO_ASID_MARK_UNUSED(ti, asid) \
150: __BITMAP_CLR((asid), &(ti)->ti_asid_bitmap)
151: #define TLBINFO_ASID_MARK_USED(ti, asid) \
152: __BITMAP_SET((asid), &(ti)->ti_asid_bitmap)
153: #define TLBINFO_ASID_INUSE_P(ti, asid) \
154: __BITMAP_ISSET((asid), &(ti)->ti_asid_bitmap)
155: #define TLBINFO_ASID_RESET(ti) \
156: do { \
157: __BITMAP_ZERO(&ti->ti_asid_bitmap); \
158: for (tlb_asid_t asid = 0; asid <= KERNEL_PID; asid++) \
159: TLBINFO_ASID_MARK_USED(ti, asid); \
160: } while (0)
161: #define TLBINFO_ASID_INITIAL_FREE(asid_max) \
162: (asid_max + 1 /* 0 */ - (1 + KERNEL_PID))
163:
1.1 christos 164: struct pmap_tlb_info pmap_tlb0_info = {
165: .ti_name = "tlb0",
166: .ti_asid_hint = KERNEL_PID + 1,
167: #ifdef PMAP_TLB_NUM_PIDS
168: .ti_asid_max = IFCONSTANT(PMAP_TLB_NUM_PIDS - 1),
1.26 jdolecek 169: .ti_asids_free = IFCONSTANT(
170: TLBINFO_ASID_INITIAL_FREE(PMAP_TLB_NUM_PIDS - 1)),
1.1 christos 171: #endif
1.26 jdolecek 172: .ti_asid_bitmap._b[0] = __BITS(0, KERNEL_PID),
1.1 christos 173: #ifdef PMAP_TLB_WIRED_UPAGES
174: .ti_wired = PMAP_TLB_WIRED_UPAGES,
175: #endif
1.5 matt 176: .ti_lock = &pmap_tlb0_lock,
1.1 christos 177: .ti_pais = LIST_HEAD_INITIALIZER(pmap_tlb0_info.ti_pais),
1.3 matt 178: #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
1.1 christos 179: .ti_tlbinvop = TLBINV_NOBODY,
180: #endif
181: };
182:
183: #undef IFCONSTANT
184:
1.3 matt 185: #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
186: struct pmap_tlb_info *pmap_tlbs[PMAP_TLB_MAX] = {
1.1 christos 187: [0] = &pmap_tlb0_info,
188: };
189: u_int pmap_ntlbs = 1;
190: #endif
191:
1.3 matt 192: #ifdef MULTIPROCESSOR
1.11 joerg 193: __unused static inline bool
1.3 matt 194: pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti)
195: {
196: #if PMAP_TLB_MAX == 1
197: return !kcpuset_iszero(pm->pm_active);
198: #else
199: return kcpuset_intersecting_p(pm->pm_active, ti->ti_kcpuset);
200: #endif
201: }
202:
203: static inline bool
204: pmap_tlb_intersecting_onproc_p(pmap_t pm, struct pmap_tlb_info *ti)
205: {
206: #if PMAP_TLB_MAX == 1
207: return !kcpuset_iszero(pm->pm_onproc);
208: #else
209: return kcpuset_intersecting_p(pm->pm_onproc, ti->ti_kcpuset);
210: #endif
211: }
212: #endif
213:
1.13 matt 214: static void
1.18 matt 215: pmap_tlb_pai_check(struct pmap_tlb_info *ti, bool locked_p)
1.13 matt 216: {
1.35 skrll 217: UVMHIST_FUNC(__func__);
1.36 skrll 218: UVMHIST_CALLARGS(maphist, "(ti=%#jx)", (uintptr_t)ti, 0, 0, 0);
1.35 skrll 219:
1.13 matt 220: #ifdef DIAGNOSTIC
221: struct pmap_asid_info *pai;
1.18 matt 222: if (!locked_p)
223: TLBINFO_LOCK(ti);
1.13 matt 224: LIST_FOREACH(pai, &ti->ti_pais, pai_link) {
225: KASSERT(pai != NULL);
226: KASSERT(PAI_PMAP(pai, ti) != pmap_kernel());
227: KASSERT(pai->pai_asid > KERNEL_PID);
228: KASSERTMSG(pai->pai_asid <= ti->ti_asid_max,
229: "pm %p asid %#x", PAI_PMAP(pai, ti), pai->pai_asid);
230: KASSERTMSG(TLBINFO_ASID_INUSE_P(ti, pai->pai_asid),
231: "pm %p asid %u", PAI_PMAP(pai, ti), pai->pai_asid);
232: #ifdef MULTIPROCESSOR
233: KASSERT(pmap_tlb_intersecting_active_p(PAI_PMAP(pai, ti), ti));
234: #endif
235: }
1.18 matt 236: if (!locked_p)
237: TLBINFO_UNLOCK(ti);
1.13 matt 238: #endif
1.36 skrll 239: UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
1.13 matt 240: }
241:
242: static void
243: pmap_tlb_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai,
1.1 christos 244: struct pmap *pm)
245: {
1.32 skrll 246: UVMHIST_FUNC(__func__);
247: UVMHIST_CALLARGS(maphist, "(ti=%#jx, pai=%#jx, pm=%#jx): asid %u",
1.33 skrll 248: (uintptr_t)ti, (uintptr_t)pai, (uintptr_t)pm, pai->pai_asid);
1.13 matt 249:
1.1 christos 250: /*
251: * We must have an ASID but it must not be onproc (on a processor).
252: */
253: KASSERT(pai->pai_asid > KERNEL_PID);
1.13 matt 254: KASSERT(pai->pai_asid <= ti->ti_asid_max);
1.1 christos 255: #if defined(MULTIPROCESSOR)
1.13 matt 256: KASSERT(pmap_tlb_intersecting_active_p(pm, ti));
1.3 matt 257: KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
1.1 christos 258: #endif
259: LIST_REMOVE(pai, pai_link);
260: #ifdef DIAGNOSTIC
261: pai->pai_link.le_prev = NULL; /* tagged as unlinked */
262: #endif
263: /*
1.5 matt 264: * If the platform has a cheap way to flush ASIDs then free the ASID
265: * back into the pool. On multiprocessor systems, we will flush the
266: * ASID from the TLB when it's allocated. That way we know the flush
267: * was always done in the correct TLB space. On uniprocessor systems,
268: * just do the flush now since we know that it has been used. This has
269: * a bit less overhead. Either way, this will mean that we will only
270: * need to flush all ASIDs if all ASIDs are in use and we need to
271: * allocate a new one.
272: */
273: if (PMAP_TLB_FLUSH_ASID_ON_RESET) {
274: #ifndef MULTIPROCESSOR
1.35 skrll 275: UVMHIST_LOG(maphist, " ... asid %u flushed", pai->pai_asid, 0,
276: 0, 0);
1.5 matt 277: tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
278: #endif
279: if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
1.35 skrll 280: UVMHIST_LOG(maphist, " ... asid marked unused",
281: pai->pai_asid, 0, 0, 0);
1.5 matt 282: TLBINFO_ASID_MARK_UNUSED(ti, pai->pai_asid);
283: ti->ti_asids_free++;
284: }
285: }
286: /*
1.1 christos 287: * Note that we don't mark the ASID as not in use in the TLB's ASID
288: * bitmap (thus it can't be allocated until the ASID space is exhausted
289: * and therefore reinitialized). We don't want to flush the TLB for
290: * entries belonging to this ASID so we will let natural TLB entry
291: * replacement flush them out of the TLB. Any new entries for this
292: * pmap will need a new ASID allocated.
293: */
294: pai->pai_asid = 0;
295:
296: #if defined(MULTIPROCESSOR)
297: /*
298: * The bits in pm_active belonging to this TLB can only be changed
299: * while this TLB's lock is held.
300: */
1.3 matt 301: #if PMAP_TLB_MAX == 1
302: kcpuset_zero(pm->pm_active);
303: #else
1.13 matt 304: kcpuset_remove(pm->pm_active, ti->ti_kcpuset);
1.3 matt 305: #endif
1.13 matt 306: KASSERT(!pmap_tlb_intersecting_active_p(pm, ti));
1.1 christos 307: #endif /* MULTIPROCESSOR */
1.13 matt 308:
309: UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
1.1 christos 310: }
311:
312: void
313: pmap_tlb_info_evcnt_attach(struct pmap_tlb_info *ti)
314: {
1.23 jdolecek 315: #if defined(MULTIPROCESSOR) && !defined(PMAP_TLB_NO_SYNCI_EVCNT)
1.1 christos 316: evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_desired,
317: EVCNT_TYPE_MISC, NULL,
318: ti->ti_name, "icache syncs desired");
319: evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_asts,
320: EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired,
321: ti->ti_name, "icache sync asts");
322: evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_all,
323: EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts,
324: ti->ti_name, "icache full syncs");
325: evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_pages,
326: EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts,
327: ti->ti_name, "icache pages synced");
328: evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_duplicate,
329: EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired,
330: ti->ti_name, "icache dup pages skipped");
331: evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_deferred,
332: EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired,
333: ti->ti_name, "icache pages deferred");
1.23 jdolecek 334: #endif /* MULTIPROCESSOR && !PMAP_TLB_NO_SYNCI_EVCNT */
1.1 christos 335: evcnt_attach_dynamic_nozero(&ti->ti_evcnt_asid_reinits,
336: EVCNT_TYPE_MISC, NULL,
337: ti->ti_name, "asid pool reinit");
338: }
339:
340: void
341: pmap_tlb_info_init(struct pmap_tlb_info *ti)
342: {
343: #if defined(MULTIPROCESSOR)
1.3 matt 344: #if PMAP_TLB_MAX == 1
345: KASSERT(ti == &pmap_tlb0_info);
346: #else
1.1 christos 347: if (ti != &pmap_tlb0_info) {
1.3 matt 348: KASSERT(pmap_ntlbs < PMAP_TLB_MAX);
1.1 christos 349:
350: KASSERT(pmap_tlbs[pmap_ntlbs] == NULL);
351:
352: ti->ti_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED);
1.26 jdolecek 353: TLBINFO_ASID_RESET(ti);
1.1 christos 354: ti->ti_asid_hint = KERNEL_PID + 1;
355: ti->ti_asid_max = pmap_tlbs[0]->ti_asid_max;
1.26 jdolecek 356: ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max);
1.19 christos 357: ti->ti_tlbinvop = TLBINV_NOBODY;
1.1 christos 358: ti->ti_victim = NULL;
1.3 matt 359: kcpuset_create(&ti->ti_kcpuset, true);
1.1 christos 360: ti->ti_index = pmap_ntlbs++;
361: ti->ti_wired = 0;
362: pmap_tlbs[ti->ti_index] = ti;
363: snprintf(ti->ti_name, sizeof(ti->ti_name), "tlb%u",
364: ti->ti_index);
365: pmap_tlb_info_evcnt_attach(ti);
1.25 jdolecek 366:
367: KASSERT(ti->ti_asid_max < PMAP_TLB_BITMAP_LENGTH);
1.1 christos 368: return;
369: }
1.3 matt 370: #endif
1.1 christos 371: #endif /* MULTIPROCESSOR */
372: KASSERT(ti == &pmap_tlb0_info);
1.5 matt 373: KASSERT(ti->ti_lock == &pmap_tlb0_lock);
1.35 skrll 374:
1.1 christos 375: mutex_init(ti->ti_lock, MUTEX_DEFAULT, IPL_SCHED);
1.3 matt 376: #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
377: kcpuset_create(&ti->ti_kcpuset, true);
1.5 matt 378: kcpuset_set(ti->ti_kcpuset, cpu_index(curcpu()));
1.3 matt 379: #endif
1.35 skrll 380:
1.1 christos 381: if (ti->ti_asid_max == 0) {
382: ti->ti_asid_max = pmap_md_tlb_asid_max();
1.26 jdolecek 383: ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max);
1.1 christos 384: }
385:
1.25 jdolecek 386: KASSERT(ti->ti_asid_max < PMAP_TLB_BITMAP_LENGTH);
1.1 christos 387: }
388:
389: #if defined(MULTIPROCESSOR)
390: void
391: pmap_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci)
392: {
393: KASSERT(!CPU_IS_PRIMARY(ci));
394: KASSERT(ci->ci_data.cpu_idlelwp != NULL);
395: KASSERT(cold);
396:
397: TLBINFO_LOCK(ti);
1.3 matt 398: #if PMAP_TLB_MAX > 1
399: kcpuset_set(ti->ti_kcpuset, cpu_index(ci));
1.5 matt 400: cpu_set_tlb_info(ci, ti);
1.3 matt 401: #endif
1.1 christos 402:
403: /*
404: * Do any MD tlb info init.
405: */
406: pmap_md_tlb_info_attach(ti, ci);
407:
408: /*
1.3 matt 409: * The kernel pmap uses the kcpuset_running set so it's always
410: * up-to-date.
1.1 christos 411: */
412: TLBINFO_UNLOCK(ti);
413: }
414: #endif /* MULTIPROCESSOR */
415:
416: #ifdef DIAGNOSTIC
417: static size_t
418: pmap_tlb_asid_count(struct pmap_tlb_info *ti)
419: {
420: size_t count = 0;
421: for (tlb_asid_t asid = 1; asid <= ti->ti_asid_max; asid++) {
1.28 jdolecek 422: if (TLBINFO_ASID_INUSE_P(ti, asid))
423: count++;
1.1 christos 424: }
425: return count;
426: }
427: #endif
428:
429: static void
430: pmap_tlb_asid_reinitialize(struct pmap_tlb_info *ti, enum tlb_invalidate_op op)
431: {
1.32 skrll 432: UVMHIST_FUNC(__func__);
433: UVMHIST_CALLARGS(maphist, "(ti=%#jx, op=%ju)", (uintptr_t)ti, op, 0, 0);
1.13 matt 434:
1.18 matt 435: pmap_tlb_pai_check(ti, true);
1.1 christos 436:
1.5 matt 437: ti->ti_evcnt_asid_reinits.ev_count++;
438:
1.1 christos 439: /*
440: * First, clear the ASID bitmap (except for ASID 0 which belongs
441: * to the kernel).
442: */
1.26 jdolecek 443: ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max);
1.1 christos 444: ti->ti_asid_hint = KERNEL_PID + 1;
1.25 jdolecek 445: TLBINFO_ASID_RESET(ti);
1.1 christos 446:
447: switch (op) {
1.13 matt 448: #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)
1.1 christos 449: case TLBINV_ALL:
450: tlb_invalidate_all();
451: break;
452: case TLBINV_ALLUSER:
453: tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max);
454: break;
1.13 matt 455: #endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */
1.1 christos 456: case TLBINV_NOBODY: {
457: /*
458: * If we are just reclaiming ASIDs in the TLB, let's go find
459: * what ASIDs are in use in the TLB. Since this is a
460: * semi-expensive operation, we don't want to do it too often.
461: * So if more half of the ASIDs are in use, we don't have
462: * enough free ASIDs so invalidate the TLB entries with ASIDs
463: * and clear the ASID bitmap. That will force everyone to
464: * allocate a new ASID.
465: */
1.13 matt 466: #if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN)
1.1 christos 467: pmap_tlb_asid_check();
1.25 jdolecek 468: const u_int asids_found = tlb_record_asids(
469: ti->ti_asid_bitmap._b, ti->ti_asid_max);
1.1 christos 470: pmap_tlb_asid_check();
1.27 jdolecek 471: #ifdef DIAGNOSTIC
472: const u_int asids_count = pmap_tlb_asid_count(ti);
473: #endif
474: KASSERTMSG(asids_found == asids_count,
475: "found %u != count %u", asids_found, asids_count);
1.1 christos 476: if (__predict_false(asids_found >= ti->ti_asid_max / 2)) {
477: tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max);
1.13 matt 478: #else /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */
1.1 christos 479: /*
1.9 skrll 480: * For those systems (PowerPC) that don't require
1.1 christos 481: * cross cpu TLB shootdowns, we have to invalidate the
482: * entire TLB because we can't record the ASIDs in use
483: * on the other CPUs. This is hopefully cheaper than
484: * than trying to use an IPI to record all the ASIDs
485: * on all the CPUs (which would be a synchronization
486: * nightmare).
487: */
488: tlb_invalidate_all();
1.13 matt 489: #endif /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */
1.25 jdolecek 490: TLBINFO_ASID_RESET(ti);
1.26 jdolecek 491: ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(
492: ti->ti_asid_max);
1.13 matt 493: #if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN)
1.1 christos 494: } else {
495: ti->ti_asids_free -= asids_found;
496: }
1.13 matt 497: #endif /* !MULTIPROCESSOR || PMAP_TLB_NEED_SHOOTDOWN */
1.5 matt 498: KASSERTMSG(ti->ti_asids_free <= ti->ti_asid_max, "%u",
499: ti->ti_asids_free);
1.1 christos 500: break;
501: }
502: default:
503: panic("%s: unexpected op %d", __func__, op);
504: }
505:
506: /*
507: * Now go through the active ASIDs. If the ASID is on a processor or
508: * we aren't invalidating all ASIDs and the TLB has an entry owned by
509: * that ASID, mark it as in use. Otherwise release the ASID.
510: */
511: struct pmap_asid_info *pai, *next;
512: for (pai = LIST_FIRST(&ti->ti_pais); pai != NULL; pai = next) {
513: struct pmap * const pm = PAI_PMAP(pai, ti);
514: next = LIST_NEXT(pai, pai_link);
515: KASSERT(pm != pmap_kernel());
516: KASSERT(pai->pai_asid > KERNEL_PID);
517: #if defined(MULTIPROCESSOR)
1.3 matt 518: if (pmap_tlb_intersecting_onproc_p(pm, ti)) {
1.1 christos 519: if (!TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
520: TLBINFO_ASID_MARK_USED(ti, pai->pai_asid);
521: ti->ti_asids_free--;
522: }
523: continue;
524: }
525: #endif /* MULTIPROCESSOR */
526: if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
527: KASSERT(op == TLBINV_NOBODY);
528: } else {
1.13 matt 529: pmap_tlb_pai_reset(ti, pai, pm);
1.1 christos 530: }
531: }
532: #ifdef DIAGNOSTIC
1.5 matt 533: size_t free_count __diagused = ti->ti_asid_max - pmap_tlb_asid_count(ti);
534: KASSERTMSG(free_count == ti->ti_asids_free,
535: "bitmap error: %zu != %u", free_count, ti->ti_asids_free);
1.1 christos 536: #endif
1.13 matt 537: UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
1.1 christos 538: }
539:
1.13 matt 540: #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)
1.14 skrll 541: #if PMAP_TLB_MAX == 1
1.3 matt 542: #error shootdown not required for single TLB systems
543: #endif
1.1 christos 544: void
545: pmap_tlb_shootdown_process(void)
546: {
547: struct cpu_info * const ci = curcpu();
1.2 matt 548: struct pmap_tlb_info * const ti = cpu_tlb_info(ci);
1.1 christos 549: #ifdef DIAGNOSTIC
550: struct pmap * const pm = curlwp->l_proc->p_vmspace->vm_map.pmap;
551: #endif
552:
553: KASSERT(cpu_intr_p());
1.38 skrll 554: KASSERTMSG(ci->ci_cpl >= IPL_SCHED, "%s: cpl (%d) < IPL_SCHED (%d)",
1.1 christos 555: __func__, ci->ci_cpl, IPL_SCHED);
556:
557: TLBINFO_LOCK(ti);
558:
559: switch (ti->ti_tlbinvop) {
560: case TLBINV_ONE: {
561: /*
562: * We only need to invalidate one user ASID.
563: */
564: struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti);
565: KASSERT(ti->ti_victim != pmap_kernel());
1.37 skrll 566: if (pmap_tlb_intersecting_onproc_p(ti->ti_victim, ti)) {
1.1 christos 567: /*
568: * The victim is an active pmap so we will just
569: * invalidate its TLB entries.
570: */
571: KASSERT(pai->pai_asid > KERNEL_PID);
572: pmap_tlb_asid_check();
573: tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
574: pmap_tlb_asid_check();
575: } else if (pai->pai_asid) {
576: /*
577: * The victim is no longer an active pmap for this TLB.
578: * So simply clear its ASID and when pmap_activate is
579: * next called for this pmap, it will allocate a new
580: * ASID.
581: */
1.3 matt 582: KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
1.13 matt 583: pmap_tlb_pai_reset(ti, pai, PAI_PMAP(pai, ti));
1.1 christos 584: }
585: break;
586: }
587: case TLBINV_ALLUSER:
588: /*
589: * Flush all user TLB entries.
590: */
591: pmap_tlb_asid_reinitialize(ti, TLBINV_ALLUSER);
592: break;
593: case TLBINV_ALLKERNEL:
594: /*
595: * We need to invalidate all global TLB entries.
596: */
597: pmap_tlb_asid_check();
598: tlb_invalidate_globals();
599: pmap_tlb_asid_check();
600: break;
601: case TLBINV_ALL:
602: /*
603: * Flush all the TLB entries (user and kernel).
604: */
605: pmap_tlb_asid_reinitialize(ti, TLBINV_ALL);
606: break;
607: case TLBINV_NOBODY:
608: /*
609: * Might be spurious or another SMT CPU sharing this TLB
610: * could have already done the work.
611: */
612: break;
613: }
614:
615: /*
616: * Indicate we are done with shutdown event.
617: */
618: ti->ti_victim = NULL;
619: ti->ti_tlbinvop = TLBINV_NOBODY;
620: TLBINFO_UNLOCK(ti);
621: }
622:
623: /*
624: * This state machine could be encoded into an array of integers but since all
625: * the values fit in 3 bits, the 5 entry "table" fits in a 16 bit value which
626: * can be loaded in a single instruction.
627: */
628: #define TLBINV_MAP(op, nobody, one, alluser, allkernel, all) \
1.39 skrll 629: (((( (nobody) << 3 * TLBINV_NOBODY) \
630: | ( (one) << 3 * TLBINV_ONE) \
631: | ( (alluser) << 3 * TLBINV_ALLUSER) \
1.40 skrll 632: | ((allkernel) << 3 * TLBINV_ALLKERNEL) \
1.39 skrll 633: | ( (all) << 3 * TLBINV_ALL)) >> 3 * (op)) & 7)
1.1 christos 634:
635: #define TLBINV_USER_MAP(op) \
636: TLBINV_MAP(op, TLBINV_ONE, TLBINV_ALLUSER, TLBINV_ALLUSER, \
637: TLBINV_ALL, TLBINV_ALL)
638:
639: #define TLBINV_KERNEL_MAP(op) \
640: TLBINV_MAP(op, TLBINV_ALLKERNEL, TLBINV_ALL, TLBINV_ALL, \
641: TLBINV_ALLKERNEL, TLBINV_ALL)
642:
643: bool
644: pmap_tlb_shootdown_bystanders(pmap_t pm)
645: {
646: /*
1.16 skrll 647: * We don't need to deal with our own TLB.
1.1 christos 648: */
1.13 matt 649:
1.32 skrll 650: UVMHIST_FUNC(__func__);
651: UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0);
1.13 matt 652:
1.34 skrll 653: const struct cpu_info * const ci = curcpu();
654: kcpuset_t *pm_active = ci->ci_shootdowncpus;
655: kcpuset_copy(pm_active, pm->pm_active);
1.13 matt 656: kcpuset_remove(pm_active, cpu_tlb_info(curcpu())->ti_kcpuset);
1.1 christos 657: const bool kernel_p = (pm == pmap_kernel());
658: bool ipi_sent = false;
659:
660: /*
661: * If pm_active gets more bits set, then it's after all our changes
662: * have been made so they will already be cognizant of them.
663: */
664:
1.3 matt 665: for (size_t i = 0; !kcpuset_iszero(pm_active); i++) {
1.1 christos 666: KASSERT(i < pmap_ntlbs);
667: struct pmap_tlb_info * const ti = pmap_tlbs[i];
668: KASSERT(tlbinfo_index(ti) == i);
669: /*
670: * Skip this TLB if there are no active mappings for it.
671: */
1.3 matt 672: if (!kcpuset_intersecting_p(pm_active, ti->ti_kcpuset))
1.1 christos 673: continue;
674: struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
1.3 matt 675: kcpuset_remove(pm_active, ti->ti_kcpuset);
1.1 christos 676: TLBINFO_LOCK(ti);
1.12 matt 677: cpuid_t j = kcpuset_ffs_intersecting(pm->pm_onproc,
678: ti->ti_kcpuset);
679: // post decrement since ffs returns bit + 1 or 0 if no bit
680: if (j-- > 0) {
1.1 christos 681: if (kernel_p) {
682: ti->ti_tlbinvop =
683: TLBINV_KERNEL_MAP(ti->ti_tlbinvop);
684: ti->ti_victim = NULL;
685: } else {
686: KASSERT(pai->pai_asid);
687: if (__predict_false(ti->ti_victim == pm)) {
688: KASSERT(ti->ti_tlbinvop == TLBINV_ONE);
689: /*
690: * We still need to invalidate this one
691: * ASID so there's nothing to change.
692: */
693: } else {
694: ti->ti_tlbinvop =
695: TLBINV_USER_MAP(ti->ti_tlbinvop);
696: if (ti->ti_tlbinvop == TLBINV_ONE)
697: ti->ti_victim = pm;
698: else
699: ti->ti_victim = NULL;
700: }
701: }
702: TLBINFO_UNLOCK(ti);
703: /*
704: * Now we can send out the shootdown IPIs to a CPU
705: * that shares this TLB and is currently using this
706: * pmap. That CPU will process the IPI and do the
707: * all the work. Any other CPUs sharing that TLB
708: * will take advantage of that work. pm_onproc might
709: * change now that we have released the lock but we
710: * can tolerate spurious shootdowns.
711: */
712: cpu_send_ipi(cpu_lookup(j), IPI_SHOOTDOWN);
713: ipi_sent = true;
714: continue;
715: }
1.3 matt 716: if (!pmap_tlb_intersecting_active_p(pm, ti)) {
1.1 christos 717: /*
718: * If this pmap has an ASID assigned but it's not
719: * currently running, nuke its ASID. Next time the
720: * pmap is activated, it will allocate a new ASID.
721: * And best of all, we avoid an IPI.
722: */
723: KASSERT(!kernel_p);
1.13 matt 724: pmap_tlb_pai_reset(ti, pai, pm);
1.1 christos 725: //ti->ti_evcnt_lazy_shots.ev_count++;
726: }
727: TLBINFO_UNLOCK(ti);
728: }
729:
1.22 pgoyette 730: UVMHIST_LOG(maphist, " <-- done (ipi_sent=%jd)", ipi_sent, 0, 0, 0);
1.13 matt 731:
1.1 christos 732: return ipi_sent;
733: }
1.13 matt 734: #endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */
1.1 christos 735:
1.43 ! skrll 736: #ifndef PMAP_HWPAGEWALKER
1.1 christos 737: int
1.13 matt 738: pmap_tlb_update_addr(pmap_t pm, vaddr_t va, pt_entry_t pte, u_int flags)
1.1 christos 739: {
1.2 matt 740: struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
1.1 christos 741: struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
742: int rv = -1;
743:
1.32 skrll 744: UVMHIST_FUNC(__func__);
745: UVMHIST_CALLARGS(maphist, " (pm=%#jx va=%#jx, pte=%#jx flags=%#jx)",
1.22 pgoyette 746: (uintptr_t)pm, va, pte_value(pte), flags);
1.13 matt 747:
1.1 christos 748: KASSERT(kpreempt_disabled());
749:
1.13 matt 750: KASSERTMSG(pte_valid_p(pte), "va %#"PRIxVADDR" %#"PRIxPTE,
751: va, pte_value(pte));
752:
1.1 christos 753: TLBINFO_LOCK(ti);
754: if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {
755: pmap_tlb_asid_check();
1.13 matt 756: rv = tlb_update_addr(va, pai->pai_asid, pte,
1.1 christos 757: (flags & PMAP_TLB_INSERT) != 0);
758: pmap_tlb_asid_check();
1.13 matt 759: UVMHIST_LOG(maphist,
1.41 skrll 760: " %jd <-- tlb_update_addr(%#jx, %#jx, %#jx, ...)",
761: rv, va, pai->pai_asid, pte_value(pte));
1.13 matt 762: KASSERTMSG((flags & PMAP_TLB_INSERT) == 0 || rv == 1,
763: "pmap %p (asid %u) va %#"PRIxVADDR" pte %#"PRIxPTE" rv %d",
764: pm, pai->pai_asid, va, pte_value(pte), rv);
765: }
766: #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)
767: if (flags & PMAP_TLB_NEED_IPI)
768: pm->pm_shootdown_pending = 1;
1.1 christos 769: #endif
770: TLBINFO_UNLOCK(ti);
771:
1.22 pgoyette 772: UVMHIST_LOG(maphist, " <-- done (rv=%jd)", rv, 0, 0, 0);
1.13 matt 773:
1.1 christos 774: return rv;
775: }
1.43 ! skrll 776: #endif /* !PMAP_HWPAGEWALKER */
1.1 christos 777:
778: void
779: pmap_tlb_invalidate_addr(pmap_t pm, vaddr_t va)
780: {
1.2 matt 781: struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
1.1 christos 782: struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
783:
1.32 skrll 784: UVMHIST_FUNC(__func__);
785: UVMHIST_CALLARGS(maphist, " (pm=%#jx va=%#jx) ti=%#jx asid=%#jx",
1.22 pgoyette 786: (uintptr_t)pm, va, (uintptr_t)ti, pai->pai_asid);
1.5 matt 787:
1.1 christos 788: KASSERT(kpreempt_disabled());
789:
790: TLBINFO_LOCK(ti);
791: if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {
792: pmap_tlb_asid_check();
1.22 pgoyette 793: UVMHIST_LOG(maphist, " invalidating %#jx asid %#jx",
1.5 matt 794: va, pai->pai_asid, 0, 0);
1.1 christos 795: tlb_invalidate_addr(va, pai->pai_asid);
796: pmap_tlb_asid_check();
797: }
1.13 matt 798: #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)
1.1 christos 799: pm->pm_shootdown_pending = 1;
800: #endif
801: TLBINFO_UNLOCK(ti);
1.5 matt 802: UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
1.1 christos 803: }
804:
805: static inline void
806: pmap_tlb_asid_alloc(struct pmap_tlb_info *ti, pmap_t pm,
807: struct pmap_asid_info *pai)
808: {
809: /*
810: * We shouldn't have an ASID assigned, and thusly must not be onproc
811: * nor active.
812: */
813: KASSERT(pm != pmap_kernel());
814: KASSERT(pai->pai_asid == 0);
815: KASSERT(pai->pai_link.le_prev == NULL);
816: #if defined(MULTIPROCESSOR)
1.5 matt 817: KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
818: KASSERT(!pmap_tlb_intersecting_active_p(pm, ti));
1.1 christos 819: #endif
820: KASSERT(ti->ti_asids_free > 0);
1.5 matt 821: KASSERT(ti->ti_asid_hint > KERNEL_PID);
822:
823: /*
824: * If the last ASID allocated was the maximum ASID, then the
825: * hint will be out of range. Reset the hint to first
826: * available ASID.
827: */
828: if (PMAP_TLB_FLUSH_ASID_ON_RESET
829: && ti->ti_asid_hint > ti->ti_asid_max) {
830: ti->ti_asid_hint = KERNEL_PID + 1;
831: }
832: KASSERTMSG(ti->ti_asid_hint <= ti->ti_asid_max, "hint %u",
833: ti->ti_asid_hint);
1.1 christos 834:
835: /*
836: * Let's see if the hinted ASID is free. If not search for
837: * a new one.
838: */
1.5 matt 839: if (__predict_true(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) {
1.25 jdolecek 840: const size_t nbpw = NBBY * sizeof(ti->ti_asid_bitmap._b[0]);
1.5 matt 841: size_t i;
842: u_long bits;
1.25 jdolecek 843: for (i = 0; (bits = ~ti->ti_asid_bitmap._b[i]) == 0; i++) {
844: KASSERT(i < __arraycount(ti->ti_asid_bitmap._b) - 1);
1.1 christos 845: }
1.5 matt 846: /*
847: * ffs wants to find the first bit set while we want
848: * to find the first bit cleared.
849: */
850: const u_int n = __builtin_ffsl(bits) - 1;
851: KASSERTMSG((bits << (nbpw - (n+1))) == (1ul << (nbpw-1)),
852: "n %u bits %#lx", n, bits);
853: KASSERT(n < nbpw);
854: ti->ti_asid_hint = n + i * nbpw;
1.1 christos 855: }
856:
1.5 matt 857: KASSERT(ti->ti_asid_hint > KERNEL_PID);
858: KASSERT(ti->ti_asid_hint <= ti->ti_asid_max);
859: KASSERTMSG(PMAP_TLB_FLUSH_ASID_ON_RESET
860: || TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint - 1),
1.25 jdolecek 861: "hint %u bitmap %p", ti->ti_asid_hint, &ti->ti_asid_bitmap);
1.5 matt 862: KASSERTMSG(!TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint),
1.25 jdolecek 863: "hint %u bitmap %p", ti->ti_asid_hint, &ti->ti_asid_bitmap);
1.5 matt 864:
1.1 christos 865: /*
866: * The hint contains our next ASID so take it and advance the hint.
867: * Mark it as used and insert the pai into the list of active asids.
868: * There is also one less asid free in this TLB.
869: */
1.5 matt 870: KASSERT(ti->ti_asid_hint > KERNEL_PID);
1.1 christos 871: pai->pai_asid = ti->ti_asid_hint++;
1.5 matt 872: #ifdef MULTIPROCESSOR
873: if (PMAP_TLB_FLUSH_ASID_ON_RESET) {
874: /*
875: * Clean the new ASID from the TLB.
876: */
877: tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
878: }
879: #endif
1.1 christos 880: TLBINFO_ASID_MARK_USED(ti, pai->pai_asid);
881: LIST_INSERT_HEAD(&ti->ti_pais, pai, pai_link);
882: ti->ti_asids_free--;
883:
884: #if defined(MULTIPROCESSOR)
885: /*
886: * Mark that we now have an active ASID for all CPUs sharing this TLB.
887: * The bits in pm_active belonging to this TLB can only be changed
888: * while this TLBs lock is held.
889: */
1.3 matt 890: #if PMAP_TLB_MAX == 1
891: kcpuset_copy(pm->pm_active, kcpuset_running);
892: #else
1.13 matt 893: kcpuset_merge(pm->pm_active, ti->ti_kcpuset);
1.3 matt 894: #endif
1.1 christos 895: #endif
896: }
897:
898: /*
899: * Acquire a TLB address space tag (called ASID or TLBPID) and return it.
900: * ASID might have already been previously acquired.
901: */
902: void
903: pmap_tlb_asid_acquire(pmap_t pm, struct lwp *l)
904: {
905: struct cpu_info * const ci = l->l_cpu;
1.2 matt 906: struct pmap_tlb_info * const ti = cpu_tlb_info(ci);
1.1 christos 907: struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
908:
1.32 skrll 909: UVMHIST_FUNC(__func__);
910: UVMHIST_CALLARGS(maphist, "(pm=%#jx, l=%#jx, ti=%#jx)", (uintptr_t)pm,
1.22 pgoyette 911: (uintptr_t)l, (uintptr_t)ti, 0);
1.13 matt 912:
1.1 christos 913: KASSERT(kpreempt_disabled());
914:
915: /*
916: * Kernels use a fixed ASID and thus doesn't need to acquire one.
917: */
1.5 matt 918: if (pm == pmap_kernel()) {
919: UVMHIST_LOG(maphist, " <-- done (kernel)", 0, 0, 0, 0);
1.1 christos 920: return;
1.5 matt 921: }
1.1 christos 922:
923: TLBINFO_LOCK(ti);
924: KASSERT(pai->pai_asid <= KERNEL_PID || pai->pai_link.le_prev != NULL);
925: KASSERT(pai->pai_asid > KERNEL_PID || pai->pai_link.le_prev == NULL);
1.18 matt 926: pmap_tlb_pai_check(ti, true);
1.1 christos 927: if (__predict_false(!PMAP_PAI_ASIDVALID_P(pai, ti))) {
928: /*
929: * If we've run out ASIDs, reinitialize the ASID space.
930: */
931: if (__predict_false(tlbinfo_noasids_p(ti))) {
932: KASSERT(l == curlwp);
1.5 matt 933: UVMHIST_LOG(maphist, " asid reinit", 0, 0, 0, 0);
1.1 christos 934: pmap_tlb_asid_reinitialize(ti, TLBINV_NOBODY);
1.5 matt 935: KASSERT(!tlbinfo_noasids_p(ti));
1.1 christos 936: }
937:
938: /*
939: * Get an ASID.
940: */
941: pmap_tlb_asid_alloc(ti, pm, pai);
1.22 pgoyette 942: UVMHIST_LOG(maphist, "allocated asid %#jx", pai->pai_asid,
943: 0, 0, 0);
1.1 christos 944: }
1.18 matt 945: pmap_tlb_pai_check(ti, true);
1.13 matt 946: #if defined(MULTIPROCESSOR)
947: KASSERT(kcpuset_isset(pm->pm_active, cpu_index(ci)));
948: #endif
1.1 christos 949:
950: if (l == curlwp) {
951: #if defined(MULTIPROCESSOR)
952: /*
953: * The bits in pm_onproc belonging to this TLB can only
954: * be changed while this TLBs lock is held unless atomic
955: * operations are used.
956: */
1.5 matt 957: KASSERT(pm != pmap_kernel());
1.3 matt 958: kcpuset_atomic_set(pm->pm_onproc, cpu_index(ci));
1.1 christos 959: #endif
960: ci->ci_pmap_asid_cur = pai->pai_asid;
1.22 pgoyette 961: UVMHIST_LOG(maphist, "setting asid to %#jx", pai->pai_asid,
962: 0, 0, 0);
1.1 christos 963: tlb_set_asid(pai->pai_asid);
964: pmap_tlb_asid_check();
965: } else {
966: printf("%s: l (%p) != curlwp %p\n", __func__, l, curlwp);
967: }
968: TLBINFO_UNLOCK(ti);
1.5 matt 969: UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
1.1 christos 970: }
971:
972: void
973: pmap_tlb_asid_deactivate(pmap_t pm)
974: {
1.32 skrll 975: UVMHIST_FUNC(__func__);
976: UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0);
1.13 matt 977:
1.1 christos 978: KASSERT(kpreempt_disabled());
979: #if defined(MULTIPROCESSOR)
980: /*
981: * The kernel pmap is aways onproc and active and must never have
982: * those bits cleared. If pmap_remove_all was called, it has already
983: * deactivated the pmap and thusly onproc will be 0 so there's nothing
984: * to do.
985: */
1.5 matt 986: if (pm != pmap_kernel() && !kcpuset_iszero(pm->pm_onproc)) {
1.1 christos 987: struct cpu_info * const ci = curcpu();
988: KASSERT(!cpu_intr_p());
1.3 matt 989: KASSERTMSG(kcpuset_isset(pm->pm_onproc, cpu_index(ci)),
990: "%s: pmap %p onproc %p doesn't include cpu %d (%p)",
1.1 christos 991: __func__, pm, pm->pm_onproc, cpu_index(ci), ci);
992: /*
993: * The bits in pm_onproc that belong to this TLB can
994: * be changed while this TLBs lock is not held as long
995: * as we use atomic ops.
996: */
1.3 matt 997: kcpuset_atomic_clear(pm->pm_onproc, cpu_index(ci));
1.1 christos 998: }
1.5 matt 999: #endif
1.17 skrll 1000: curcpu()->ci_pmap_asid_cur = KERNEL_PID;
1.10 skrll 1001: tlb_set_asid(KERNEL_PID);
1.35 skrll 1002:
1.18 matt 1003: pmap_tlb_pai_check(cpu_tlb_info(curcpu()), false);
1.5 matt 1004: #if defined(DEBUG)
1.1 christos 1005: pmap_tlb_asid_check();
1006: #endif
1.35 skrll 1007: UVMHIST_LOG(maphist, " <-- done (pm=%#jx)", (uintptr_t)pm, 0, 0, 0);
1.1 christos 1008: }
1009:
1010: void
1011: pmap_tlb_asid_release_all(struct pmap *pm)
1012: {
1.32 skrll 1013: UVMHIST_FUNC(__func__);
1014: UVMHIST_CALLARGS(maphist, "(pm=%#jx)", (uintptr_t)pm, 0, 0, 0);
1.13 matt 1015:
1.1 christos 1016: KASSERT(pm != pmap_kernel());
1017: #if defined(MULTIPROCESSOR)
1.5 matt 1018: //KASSERT(!kcpuset_iszero(pm->pm_onproc)); // XXX
1.13 matt 1019: struct cpu_info * const ci __diagused = curcpu();
1020: KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci)));
1.8 matt 1021: #if PMAP_TLB_MAX > 1
1.3 matt 1022: for (u_int i = 0; !kcpuset_iszero(pm->pm_active); i++) {
1.1 christos 1023: KASSERT(i < pmap_ntlbs);
1024: struct pmap_tlb_info * const ti = pmap_tlbs[i];
1.3 matt 1025: #else
1026: struct pmap_tlb_info * const ti = &pmap_tlb0_info;
1027: #endif
1.5 matt 1028: struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
1029: TLBINFO_LOCK(ti);
1030: if (PMAP_PAI_ASIDVALID_P(pai, ti)) {
1031: /*
1.13 matt 1032: * This pmap should not be in use by any other cpu so
1033: * we can just reset and be happy.
1.5 matt 1034: */
1.13 matt 1035: if (ti->ti_victim == pm)
1036: ti->ti_victim = NULL;
1037: pmap_tlb_pai_reset(ti, pai, pm);
1.1 christos 1038: }
1.13 matt 1039: KASSERT(pai->pai_link.le_prev == NULL);
1.5 matt 1040: TLBINFO_UNLOCK(ti);
1.3 matt 1041: #if PMAP_TLB_MAX > 1
1.1 christos 1042: }
1.3 matt 1043: #endif
1.13 matt 1044: #ifdef DIAGNOSTIC
1045: for (size_t i = 0; i < (PMAP_TLB_MAX > 1 ? pmap_ntlbs : 1); i++) {
1046: KASSERTMSG(pm->pm_pai[i].pai_asid == 0,
1047: "pm %p i %zu asid %u",
1048: pm, i, pm->pm_pai[i].pai_asid);
1049: }
1050: #endif
1.1 christos 1051: #else
1052: /*
1.16 skrll 1053: * Handle the case of an UP kernel which only has, at most, one TLB.
1.1 christos 1054: * If the pmap has an ASID allocated, free it.
1055: */
1056: struct pmap_tlb_info * const ti = &pmap_tlb0_info;
1057: struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
1058: TLBINFO_LOCK(ti);
1059: if (pai->pai_asid > KERNEL_PID) {
1.7 matt 1060: if (curcpu()->ci_pmap_asid_cur == pai->pai_asid) {
1.5 matt 1061: tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
1062: } else {
1.13 matt 1063: pmap_tlb_pai_reset(ti, pai, pm);
1.5 matt 1064: }
1.1 christos 1065: }
1066: TLBINFO_UNLOCK(ti);
1067: #endif /* MULTIPROCESSOR */
1.13 matt 1068: UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
1.1 christos 1069: }
1070:
1071: void
1072: pmap_tlb_asid_check(void)
1073: {
1074: #ifdef DEBUG
1075: kpreempt_disable();
1.5 matt 1076: const tlb_asid_t asid __debugused = tlb_get_asid();
1.1 christos 1077: KDASSERTMSG(asid == curcpu()->ci_pmap_asid_cur,
1078: "%s: asid (%#x) != current asid (%#x)",
1079: __func__, asid, curcpu()->ci_pmap_asid_cur);
1080: kpreempt_enable();
1081: #endif
1082: }
1083:
1084: #ifdef DEBUG
1085: void
1086: pmap_tlb_check(pmap_t pm, bool (*func)(void *, vaddr_t, tlb_asid_t, pt_entry_t))
1087: {
1.2 matt 1088: struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
1.1 christos 1089: struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
1090: TLBINFO_LOCK(ti);
1091: if (pm == pmap_kernel() || pai->pai_asid > KERNEL_PID)
1092: tlb_walk(pm, func);
1093: TLBINFO_UNLOCK(ti);
1094: }
1095: #endif /* DEBUG */
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to pmap_tlb.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / uvm / pmap