version 1.10.2.5, 2017/08/28 17:53:18 |
version 1.11, 2015/04/18 16:58:31 |
Line 181 u_int pmap_ntlbs = 1; |
|
Line 181 u_int pmap_ntlbs = 1; |
|
#define TLBINFO_ASID_INUSE_P(ti, asid) \ |
#define TLBINFO_ASID_INUSE_P(ti, asid) \ |
__BITMAP_ISSET_P((ti)->ti_asid_bitmap, (asid)) |
__BITMAP_ISSET_P((ti)->ti_asid_bitmap, (asid)) |
|
|
|
static void |
|
pmap_pai_check(struct pmap_tlb_info *ti) |
|
{ |
|
#ifdef DIAGNOSTIC |
|
struct pmap_asid_info *pai; |
|
LIST_FOREACH(pai, &ti->ti_pais, pai_link) { |
|
KASSERT(pai != NULL); |
|
KASSERT(PAI_PMAP(pai, ti) != pmap_kernel()); |
|
KASSERT(pai->pai_asid > KERNEL_PID); |
|
KASSERT(TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)); |
|
} |
|
#endif |
|
} |
|
|
#ifdef MULTIPROCESSOR |
#ifdef MULTIPROCESSOR |
__unused static inline bool |
__unused static inline bool |
pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti) |
pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti) |
Line 203 pmap_tlb_intersecting_onproc_p(pmap_t pm |
|
Line 217 pmap_tlb_intersecting_onproc_p(pmap_t pm |
|
} |
} |
#endif |
#endif |
|
|
static void |
static inline void |
pmap_tlb_pai_check(struct pmap_tlb_info *ti, bool locked_p) |
pmap_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai, |
{ |
|
#ifdef DIAGNOSTIC |
|
struct pmap_asid_info *pai; |
|
if (!locked_p) |
|
TLBINFO_LOCK(ti); |
|
LIST_FOREACH(pai, &ti->ti_pais, pai_link) { |
|
KASSERT(pai != NULL); |
|
KASSERT(PAI_PMAP(pai, ti) != pmap_kernel()); |
|
KASSERT(pai->pai_asid > KERNEL_PID); |
|
KASSERTMSG(pai->pai_asid <= ti->ti_asid_max, |
|
"pm %p asid %#x", PAI_PMAP(pai, ti), pai->pai_asid); |
|
KASSERTMSG(TLBINFO_ASID_INUSE_P(ti, pai->pai_asid), |
|
"pm %p asid %u", PAI_PMAP(pai, ti), pai->pai_asid); |
|
#ifdef MULTIPROCESSOR |
|
KASSERT(pmap_tlb_intersecting_active_p(PAI_PMAP(pai, ti), ti)); |
|
#endif |
|
} |
|
if (!locked_p) |
|
TLBINFO_UNLOCK(ti); |
|
#endif |
|
} |
|
|
|
static void |
|
pmap_tlb_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai, |
|
struct pmap *pm) |
struct pmap *pm) |
{ |
{ |
UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); |
|
UVMHIST_LOG(maphist, "(ti=%p, pai=%p, pm=%p): asid %u", |
|
ti, pai, pm, pai->pai_asid); |
|
|
|
/* |
/* |
* We must have an ASID but it must not be onproc (on a processor). |
* We must have an ASID but it must not be onproc (on a processor). |
*/ |
*/ |
KASSERT(pai->pai_asid > KERNEL_PID); |
KASSERT(pai->pai_asid > KERNEL_PID); |
KASSERT(pai->pai_asid <= ti->ti_asid_max); |
|
#if defined(MULTIPROCESSOR) |
#if defined(MULTIPROCESSOR) |
KASSERT(pmap_tlb_intersecting_active_p(pm, ti)); |
|
KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); |
KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); |
#endif |
#endif |
LIST_REMOVE(pai, pai_link); |
LIST_REMOVE(pai, pai_link); |
Line 285 pmap_tlb_pai_reset(struct pmap_tlb_info |
|
Line 269 pmap_tlb_pai_reset(struct pmap_tlb_info |
|
#if PMAP_TLB_MAX == 1 |
#if PMAP_TLB_MAX == 1 |
kcpuset_zero(pm->pm_active); |
kcpuset_zero(pm->pm_active); |
#else |
#else |
kcpuset_remove(pm->pm_active, ti->ti_kcpuset); |
kcpuset_atomicly_remove(pm->pm_active, ti->ti_kcpuset); |
#endif |
#endif |
KASSERT(!pmap_tlb_intersecting_active_p(pm, ti)); |
|
#endif /* MULTIPROCESSOR */ |
#endif /* MULTIPROCESSOR */ |
|
|
UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); |
|
} |
} |
|
|
void |
void |
Line 338 pmap_tlb_info_init(struct pmap_tlb_info |
|
Line 319 pmap_tlb_info_init(struct pmap_tlb_info |
|
ti->ti_asid_hint = KERNEL_PID + 1; |
ti->ti_asid_hint = KERNEL_PID + 1; |
ti->ti_asid_max = pmap_tlbs[0]->ti_asid_max; |
ti->ti_asid_max = pmap_tlbs[0]->ti_asid_max; |
ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID; |
ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID; |
ti->ti_tlbinvop = TLBINV_NOBODY; |
ti->ti_tlbinvop = TLBINV_NOBODY, |
ti->ti_victim = NULL; |
ti->ti_victim = NULL; |
kcpuset_create(&ti->ti_kcpuset, true); |
kcpuset_create(&ti->ti_kcpuset, true); |
ti->ti_index = pmap_ntlbs++; |
ti->ti_index = pmap_ntlbs++; |
Line 413 pmap_tlb_asid_reinitialize(struct pmap_t |
|
Line 394 pmap_tlb_asid_reinitialize(struct pmap_t |
|
const size_t asid_bitmap_words = |
const size_t asid_bitmap_words = |
ti->ti_asid_max / (8 * sizeof(ti->ti_asid_bitmap[0])); |
ti->ti_asid_max / (8 * sizeof(ti->ti_asid_bitmap[0])); |
|
|
UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); |
pmap_pai_check(ti); |
UVMHIST_LOG(maphist, "(ti=%p, op=%u)", ti, op, 0, 0); |
|
|
|
pmap_tlb_pai_check(ti, true); |
|
|
|
ti->ti_evcnt_asid_reinits.ev_count++; |
ti->ti_evcnt_asid_reinits.ev_count++; |
|
|
Line 432 pmap_tlb_asid_reinitialize(struct pmap_t |
|
Line 410 pmap_tlb_asid_reinitialize(struct pmap_t |
|
} |
} |
|
|
switch (op) { |
switch (op) { |
#if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) |
#if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN) |
case TLBINV_ALL: |
case TLBINV_ALL: |
tlb_invalidate_all(); |
tlb_invalidate_all(); |
break; |
break; |
case TLBINV_ALLUSER: |
case TLBINV_ALLUSER: |
tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max); |
tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max); |
break; |
break; |
#endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */ |
#endif /* MULTIPROCESSOR && PMAP_NEED_TLB_SHOOTDOWN */ |
case TLBINV_NOBODY: { |
case TLBINV_NOBODY: { |
/* |
/* |
* If we are just reclaiming ASIDs in the TLB, let's go find |
* If we are just reclaiming ASIDs in the TLB, let's go find |
Line 450 pmap_tlb_asid_reinitialize(struct pmap_t |
|
Line 428 pmap_tlb_asid_reinitialize(struct pmap_t |
|
* and clear the ASID bitmap. That will force everyone to |
* and clear the ASID bitmap. That will force everyone to |
* allocate a new ASID. |
* allocate a new ASID. |
*/ |
*/ |
#if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN) |
#if !defined(MULTIPROCESSOR) || defined(PMAP_NEED_TLB_SHOOTDOWN) |
pmap_tlb_asid_check(); |
pmap_tlb_asid_check(); |
const u_int asids_found = tlb_record_asids(ti->ti_asid_bitmap, |
const u_int asids_found = tlb_record_asids(ti->ti_asid_bitmap); |
ti->ti_asid_max); |
|
pmap_tlb_asid_check(); |
pmap_tlb_asid_check(); |
KASSERT(asids_found == pmap_tlb_asid_count(ti)); |
KASSERT(asids_found == pmap_tlb_asid_count(ti)); |
if (__predict_false(asids_found >= ti->ti_asid_max / 2)) { |
if (__predict_false(asids_found >= ti->ti_asid_max / 2)) { |
tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max); |
tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max); |
#else /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */ |
#else /* MULTIPROCESSOR && !PMAP_NEED_TLB_SHOOTDOWN */ |
/* |
/* |
* For those systems (PowerPC) that don't require |
* For those systems (PowerPC) that don't require |
* cross cpu TLB shootdowns, we have to invalidate the |
* cross cpu TLB shootdowns, we have to invalidate the |
Line 469 pmap_tlb_asid_reinitialize(struct pmap_t |
|
Line 446 pmap_tlb_asid_reinitialize(struct pmap_t |
|
* nightmare). |
* nightmare). |
*/ |
*/ |
tlb_invalidate_all(); |
tlb_invalidate_all(); |
#endif /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */ |
#endif /* MULTIPROCESSOR && !PMAP_NEED_TLB_SHOOTDOWN */ |
ti->ti_asid_bitmap[0] = (2 << KERNEL_PID) - 1; |
ti->ti_asid_bitmap[0] = (2 << KERNEL_PID) - 1; |
for (size_t word = 1; |
for (size_t word = 1; |
word <= asid_bitmap_words; |
word <= asid_bitmap_words; |
Line 477 pmap_tlb_asid_reinitialize(struct pmap_t |
|
Line 454 pmap_tlb_asid_reinitialize(struct pmap_t |
|
ti->ti_asid_bitmap[word] = 0; |
ti->ti_asid_bitmap[word] = 0; |
} |
} |
ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID; |
ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID; |
#if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN) |
#if !defined(MULTIPROCESSOR) || defined(PMAP_NEED_TLB_SHOOTDOWN) |
} else { |
} else { |
ti->ti_asids_free -= asids_found; |
ti->ti_asids_free -= asids_found; |
} |
} |
#endif /* !MULTIPROCESSOR || PMAP_TLB_NEED_SHOOTDOWN */ |
#endif /* !MULTIPROCESSOR || PMAP_NEED_TLB_SHOOTDOWN */ |
KASSERTMSG(ti->ti_asids_free <= ti->ti_asid_max, "%u", |
KASSERTMSG(ti->ti_asids_free <= ti->ti_asid_max, "%u", |
ti->ti_asids_free); |
ti->ti_asids_free); |
break; |
break; |
Line 513 pmap_tlb_asid_reinitialize(struct pmap_t |
|
Line 490 pmap_tlb_asid_reinitialize(struct pmap_t |
|
if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { |
if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { |
KASSERT(op == TLBINV_NOBODY); |
KASSERT(op == TLBINV_NOBODY); |
} else { |
} else { |
pmap_tlb_pai_reset(ti, pai, pm); |
pmap_pai_reset(ti, pai, pm); |
} |
} |
} |
} |
#ifdef DIAGNOSTIC |
#ifdef DIAGNOSTIC |
Line 521 pmap_tlb_asid_reinitialize(struct pmap_t |
|
Line 498 pmap_tlb_asid_reinitialize(struct pmap_t |
|
KASSERTMSG(free_count == ti->ti_asids_free, |
KASSERTMSG(free_count == ti->ti_asids_free, |
"bitmap error: %zu != %u", free_count, ti->ti_asids_free); |
"bitmap error: %zu != %u", free_count, ti->ti_asids_free); |
#endif |
#endif |
UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); |
|
} |
} |
|
|
#if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) |
#if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN) |
#if PMAP_TLB_MAX == 1 |
#if PMAP_MAX_TLB == 1 |
#error shootdown not required for single TLB systems |
#error shootdown not required for single TLB systems |
#endif |
#endif |
void |
void |
Line 551 pmap_tlb_shootdown_process(void) |
|
Line 527 pmap_tlb_shootdown_process(void) |
|
*/ |
*/ |
struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti); |
struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti); |
KASSERT(ti->ti_victim != pmap_kernel()); |
KASSERT(ti->ti_victim != pmap_kernel()); |
if (!pmap_tlb_intersecting_onproc_p(ti->ti_victim, ti)) { |
if (!pmap_tlb_intersecting_onproc_p(ti_victim->pm_onproc, ti)) { |
/* |
/* |
* The victim is an active pmap so we will just |
* The victim is an active pmap so we will just |
* invalidate its TLB entries. |
* invalidate its TLB entries. |
Line 568 pmap_tlb_shootdown_process(void) |
|
Line 544 pmap_tlb_shootdown_process(void) |
|
* ASID. |
* ASID. |
*/ |
*/ |
KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); |
KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); |
pmap_tlb_pai_reset(ti, pai, PAI_PMAP(pai, ti)); |
pmap_pai_reset(ti, pai, PAI_PMAP(pai, ti)); |
} |
} |
break; |
break; |
} |
} |
|
|
pmap_tlb_shootdown_bystanders(pmap_t pm) |
pmap_tlb_shootdown_bystanders(pmap_t pm) |
{ |
{ |
/* |
/* |
* We don't need to deal with our own TLB. |
* We don't need to deal our own TLB. |
*/ |
*/ |
|
|
UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); |
|
|
|
kcpuset_t *pm_active; |
kcpuset_t *pm_active; |
|
|
kcpuset_clone(&pm_active, pm->pm_active); |
kcpuset_clone(&pm_active, pm->pm_active); |
kcpuset_remove(pm_active, cpu_tlb_info(curcpu())->ti_kcpuset); |
kcpuset_atomicly_remove(pm->pm_active, |
|
cpu_tlb_info(curcpu())->ti_kcpuset); |
const bool kernel_p = (pm == pmap_kernel()); |
const bool kernel_p = (pm == pmap_kernel()); |
bool ipi_sent = false; |
bool ipi_sent = false; |
|
|
Line 660 pmap_tlb_shootdown_bystanders(pmap_t pm) |
|
Line 635 pmap_tlb_shootdown_bystanders(pmap_t pm) |
|
struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); |
struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); |
kcpuset_remove(pm_active, ti->ti_kcpuset); |
kcpuset_remove(pm_active, ti->ti_kcpuset); |
TLBINFO_LOCK(ti); |
TLBINFO_LOCK(ti); |
cpuid_t j = kcpuset_ffs_intersecting(pm->pm_onproc, |
if (pmap_tlb_intersecting_onproc_p(pm, ti)) { |
ti->ti_kcpuset); |
cpuid_t j = kcpuset_ffs_intersecting(pm->pm_onproc, |
// post decrement since ffs returns bit + 1 or 0 if no bit |
ti->ti_kcpuset); |
if (j-- > 0) { |
|
if (kernel_p) { |
if (kernel_p) { |
ti->ti_tlbinvop = |
ti->ti_tlbinvop = |
TLBINV_KERNEL_MAP(ti->ti_tlbinvop); |
TLBINV_KERNEL_MAP(ti->ti_tlbinvop); |
Line 707 pmap_tlb_shootdown_bystanders(pmap_t pm) |
|
Line 681 pmap_tlb_shootdown_bystanders(pmap_t pm) |
|
* And best of all, we avoid an IPI. |
* And best of all, we avoid an IPI. |
*/ |
*/ |
KASSERT(!kernel_p); |
KASSERT(!kernel_p); |
pmap_tlb_pai_reset(ti, pai, pm); |
pmap_pai_reset(ti, pai, pm); |
//ti->ti_evcnt_lazy_shots.ev_count++; |
//ti->ti_evcnt_lazy_shots.ev_count++; |
} |
} |
TLBINFO_UNLOCK(ti); |
TLBINFO_UNLOCK(ti); |
Line 715 pmap_tlb_shootdown_bystanders(pmap_t pm) |
|
Line 689 pmap_tlb_shootdown_bystanders(pmap_t pm) |
|
|
|
kcpuset_destroy(pm_active); |
kcpuset_destroy(pm_active); |
|
|
UVMHIST_LOG(maphist, " <-- done (ipi_sent=%d)", ipi_sent, 0, 0, 0); |
|
|
|
return ipi_sent; |
return ipi_sent; |
} |
} |
#endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */ |
#endif /* MULTIPROCESSOR && PMAP_NEED_TLB_SHOOTDOWN */ |
|
|
#ifndef PMAP_HWPAGEWALKER |
#ifndef PMAP_TLB_HWPAGEWALKER |
int |
int |
pmap_tlb_update_addr(pmap_t pm, vaddr_t va, pt_entry_t pte, u_int flags) |
pmap_tlb_update_addr(pmap_t pm, vaddr_t va, pt_entry_t pt_entry, u_int flags) |
{ |
{ |
struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); |
struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); |
struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); |
struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); |
int rv = -1; |
int rv = -1; |
|
|
UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); |
|
UVMHIST_LOG(maphist, |
|
" (pm=%p va=%#"PRIxVADDR", pte=%#"PRIxPTE" flags=%#x)", |
|
pm, va, pte_value(pte), flags); |
|
|
|
KASSERT(kpreempt_disabled()); |
KASSERT(kpreempt_disabled()); |
|
|
KASSERTMSG(pte_valid_p(pte), "va %#"PRIxVADDR" %#"PRIxPTE, |
|
va, pte_value(pte)); |
|
|
|
TLBINFO_LOCK(ti); |
TLBINFO_LOCK(ti); |
if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { |
if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { |
pmap_tlb_asid_check(); |
pmap_tlb_asid_check(); |
rv = tlb_update_addr(va, pai->pai_asid, pte, |
rv = tlb_update_addr(va, pai->pai_asid, pt_entry, |
(flags & PMAP_TLB_INSERT) != 0); |
(flags & PMAP_TLB_INSERT) != 0); |
pmap_tlb_asid_check(); |
pmap_tlb_asid_check(); |
UVMHIST_LOG(maphist, |
} |
" %d <-- tlb_update_addr(%#"PRIxVADDR", %#x, %#"PRIxPTE", ...)", |
#if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN) |
rv, va, pai->pai_asid, pte_value(pte)); |
pm->pm_shootdown_pending = (flags & PMAP_TLB_NEED_IPI) != 0; |
KASSERTMSG((flags & PMAP_TLB_INSERT) == 0 || rv == 1, |
|
"pmap %p (asid %u) va %#"PRIxVADDR" pte %#"PRIxPTE" rv %d", |
|
pm, pai->pai_asid, va, pte_value(pte), rv); |
|
} |
|
#if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) |
|
if (flags & PMAP_TLB_NEED_IPI) |
|
pm->pm_shootdown_pending = 1; |
|
#endif |
#endif |
TLBINFO_UNLOCK(ti); |
TLBINFO_UNLOCK(ti); |
|
|
UVMHIST_LOG(maphist, " <-- done (rv=%d)", rv, 0, 0, 0); |
|
|
|
return rv; |
return rv; |
} |
} |
#endif /* !PMAP_HWPAGEWALKER */ |
#endif /* !PMAP_TLB_HWPAGEWALKER */ |
|
|
void |
void |
pmap_tlb_invalidate_addr(pmap_t pm, vaddr_t va) |
pmap_tlb_invalidate_addr(pmap_t pm, vaddr_t va) |
Line 771 pmap_tlb_invalidate_addr(pmap_t pm, vadd |
|
Line 726 pmap_tlb_invalidate_addr(pmap_t pm, vadd |
|
struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); |
struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); |
|
|
UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); |
UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); |
UVMHIST_LOG(maphist, " (pm=%p va=%#"PRIxVADDR") ti=%p asid=%#x", |
|
pm, va, ti, pai->pai_asid); |
|
|
|
KASSERT(kpreempt_disabled()); |
KASSERT(kpreempt_disabled()); |
|
|
|
UVMHIST_LOG(maphist, " (pm=%#x va=%#x) ti=%#x asid=%#x", |
|
pm, va, ti, pai->pai_asid); |
|
|
TLBINFO_LOCK(ti); |
TLBINFO_LOCK(ti); |
if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { |
if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { |
pmap_tlb_asid_check(); |
pmap_tlb_asid_check(); |
UVMHIST_LOG(maphist, " invalidating %#"PRIxVADDR" asid %#x", |
UVMHIST_LOG(maphist, " invalidating %#x asid %#x", |
va, pai->pai_asid, 0, 0); |
va, pai->pai_asid, 0, 0); |
tlb_invalidate_addr(va, pai->pai_asid); |
tlb_invalidate_addr(va, pai->pai_asid); |
pmap_tlb_asid_check(); |
pmap_tlb_asid_check(); |
} |
} |
#if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) |
#if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN) |
pm->pm_shootdown_pending = 1; |
pm->pm_shootdown_pending = 1; |
#endif |
#endif |
TLBINFO_UNLOCK(ti); |
TLBINFO_UNLOCK(ti); |
Line 826 pmap_tlb_asid_alloc(struct pmap_tlb_info |
|
Line 782 pmap_tlb_asid_alloc(struct pmap_tlb_info |
|
* a new one. |
* a new one. |
*/ |
*/ |
if (__predict_true(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) { |
if (__predict_true(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) { |
const size_t nbpw = 8 * sizeof(ti->ti_asid_bitmap[0]); |
const size_t nbpw __diagused = 8*sizeof(ti->ti_asid_bitmap[0]); |
size_t i; |
size_t i; |
u_long bits; |
u_long bits; |
for (i = 0; (bits = ~ti->ti_asid_bitmap[i]) == 0; i++) { |
for (i = 0; (bits = ~ti->ti_asid_bitmap[i]) == 0; i++) { |
Line 879 pmap_tlb_asid_alloc(struct pmap_tlb_info |
|
Line 835 pmap_tlb_asid_alloc(struct pmap_tlb_info |
|
#if PMAP_TLB_MAX == 1 |
#if PMAP_TLB_MAX == 1 |
kcpuset_copy(pm->pm_active, kcpuset_running); |
kcpuset_copy(pm->pm_active, kcpuset_running); |
#else |
#else |
kcpuset_merge(pm->pm_active, ti->ti_kcpuset); |
kcpuset_atomicly_merge(pm->pm_active, ti->ti_kcpuset); |
#endif |
#endif |
#endif |
#endif |
} |
} |
Line 895 pmap_tlb_asid_acquire(pmap_t pm, struct |
|
Line 851 pmap_tlb_asid_acquire(pmap_t pm, struct |
|
struct pmap_tlb_info * const ti = cpu_tlb_info(ci); |
struct pmap_tlb_info * const ti = cpu_tlb_info(ci); |
struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); |
struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); |
|
|
UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); |
|
UVMHIST_LOG(maphist, "(pm=%p, l=%p, ti=%p)", pm, l, ti, 0); |
|
|
|
KASSERT(kpreempt_disabled()); |
KASSERT(kpreempt_disabled()); |
|
|
|
UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); |
|
|
/* |
/* |
* Kernels use a fixed ASID and thus doesn't need to acquire one. |
* Kernels use a fixed ASID and thus doesn't need to acquire one. |
*/ |
*/ |
Line 908 pmap_tlb_asid_acquire(pmap_t pm, struct |
|
Line 863 pmap_tlb_asid_acquire(pmap_t pm, struct |
|
return; |
return; |
} |
} |
|
|
|
UVMHIST_LOG(maphist, " (pm=%#x, l=%#x, ti=%#x)", pm, l, ti, 0); |
TLBINFO_LOCK(ti); |
TLBINFO_LOCK(ti); |
KASSERT(pai->pai_asid <= KERNEL_PID || pai->pai_link.le_prev != NULL); |
KASSERT(pai->pai_asid <= KERNEL_PID || pai->pai_link.le_prev != NULL); |
KASSERT(pai->pai_asid > KERNEL_PID || pai->pai_link.le_prev == NULL); |
KASSERT(pai->pai_asid > KERNEL_PID || pai->pai_link.le_prev == NULL); |
pmap_tlb_pai_check(ti, true); |
pmap_pai_check(ti); |
if (__predict_false(!PMAP_PAI_ASIDVALID_P(pai, ti))) { |
if (__predict_false(!PMAP_PAI_ASIDVALID_P(pai, ti))) { |
/* |
/* |
* If we've run out ASIDs, reinitialize the ASID space. |
* If we've run out ASIDs, reinitialize the ASID space. |
Line 929 pmap_tlb_asid_acquire(pmap_t pm, struct |
|
Line 885 pmap_tlb_asid_acquire(pmap_t pm, struct |
|
pmap_tlb_asid_alloc(ti, pm, pai); |
pmap_tlb_asid_alloc(ti, pm, pai); |
UVMHIST_LOG(maphist, "allocated asid %#x", pai->pai_asid, 0, 0, 0); |
UVMHIST_LOG(maphist, "allocated asid %#x", pai->pai_asid, 0, 0, 0); |
} |
} |
pmap_tlb_pai_check(ti, true); |
|
#if defined(MULTIPROCESSOR) |
|
KASSERT(kcpuset_isset(pm->pm_active, cpu_index(ci))); |
|
#endif |
|
|
|
if (l == curlwp) { |
if (l == curlwp) { |
#if defined(MULTIPROCESSOR) |
#if defined(MULTIPROCESSOR) |
|
|
pmap_tlb_asid_deactivate(pmap_t pm) |
pmap_tlb_asid_deactivate(pmap_t pm) |
{ |
{ |
UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); |
UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); |
|
|
KASSERT(kpreempt_disabled()); |
KASSERT(kpreempt_disabled()); |
#if defined(MULTIPROCESSOR) |
#if defined(MULTIPROCESSOR) |
/* |
/* |
Line 982 pmap_tlb_asid_deactivate(pmap_t pm) |
|
Line 933 pmap_tlb_asid_deactivate(pmap_t pm) |
|
kcpuset_atomic_clear(pm->pm_onproc, cpu_index(ci)); |
kcpuset_atomic_clear(pm->pm_onproc, cpu_index(ci)); |
} |
} |
#endif |
#endif |
curcpu()->ci_pmap_asid_cur = KERNEL_PID; |
curcpu()->ci_pmap_asid_cur = 0; |
UVMHIST_LOG(maphist, " <-- done (pm=%p)", pm, 0, 0, 0); |
UVMHIST_LOG(maphist, " <-- done (pm=%#x)", pm, 0, 0, 0); |
tlb_set_asid(KERNEL_PID); |
tlb_set_asid(KERNEL_PID); |
pmap_tlb_pai_check(cpu_tlb_info(curcpu()), false); |
|
#if defined(DEBUG) |
#if defined(DEBUG) |
pmap_tlb_asid_check(); |
pmap_tlb_asid_check(); |
#endif |
#endif |
Line 994 pmap_tlb_asid_deactivate(pmap_t pm) |
|
Line 944 pmap_tlb_asid_deactivate(pmap_t pm) |
|
void |
void |
pmap_tlb_asid_release_all(struct pmap *pm) |
pmap_tlb_asid_release_all(struct pmap *pm) |
{ |
{ |
UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); |
|
UVMHIST_LOG(maphist, "(pm=%p)", pm, 0, 0, 0); |
|
|
|
KASSERT(pm != pmap_kernel()); |
KASSERT(pm != pmap_kernel()); |
#if defined(MULTIPROCESSOR) |
#if defined(MULTIPROCESSOR) |
//KASSERT(!kcpuset_iszero(pm->pm_onproc)); // XXX |
//KASSERT(!kcpuset_iszero(pm->pm_onproc)); // XXX |
struct cpu_info * const ci __diagused = curcpu(); |
|
KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci))); |
|
#if PMAP_TLB_MAX > 1 |
#if PMAP_TLB_MAX > 1 |
|
struct cpu_info * const ci __diagused = curcpu(); |
for (u_int i = 0; !kcpuset_iszero(pm->pm_active); i++) { |
for (u_int i = 0; !kcpuset_iszero(pm->pm_active); i++) { |
KASSERT(i < pmap_ntlbs); |
KASSERT(i < pmap_ntlbs); |
struct pmap_tlb_info * const ti = pmap_tlbs[i]; |
struct pmap_tlb_info * const ti = pmap_tlbs[i]; |
Line 1013 pmap_tlb_asid_release_all(struct pmap *p |
|
Line 959 pmap_tlb_asid_release_all(struct pmap *p |
|
TLBINFO_LOCK(ti); |
TLBINFO_LOCK(ti); |
if (PMAP_PAI_ASIDVALID_P(pai, ti)) { |
if (PMAP_PAI_ASIDVALID_P(pai, ti)) { |
/* |
/* |
* This pmap should not be in use by any other cpu so |
* If this pmap isn't onproc on any of the cpus |
* we can just reset and be happy. |
* belonging to this tlb domain, we can just reset |
|
* the ASID and be done. |
*/ |
*/ |
if (ti->ti_victim == pm) |
if (!pmap_tlb_intersecting_onproc_p(pm, ti)) { |
ti->ti_victim = NULL; |
KASSERT(ti->ti_victim != pm); |
pmap_tlb_pai_reset(ti, pai, pm); |
pmap_pai_reset(ti, pai, pm); |
|
#if PMAP_TLB_MAX == 1 |
|
} else { |
|
KASSERT(cpu_tlb_info(ci) == ti); |
|
tlb_invalidate_asids(pai->pai_asid, |
|
pai->pai_asid); |
|
#else |
|
} else if (cpu_tlb_info(ci) == ti) { |
|
tlb_invalidate_asids(pai->pai_asid, |
|
pai->pai_asid); |
|
} else { |
|
pm->pm_shootdown_needed = 1; |
|
#endif |
|
} |
} |
} |
KASSERT(pai->pai_link.le_prev == NULL); |
|
TLBINFO_UNLOCK(ti); |
TLBINFO_UNLOCK(ti); |
#if PMAP_TLB_MAX > 1 |
#if PMAP_TLB_MAX > 1 |
} |
} |
#endif |
#endif |
#ifdef DIAGNOSTIC |
|
for (size_t i = 0; i < (PMAP_TLB_MAX > 1 ? pmap_ntlbs : 1); i++) { |
|
KASSERTMSG(pm->pm_pai[i].pai_asid == 0, |
|
"pm %p i %zu asid %u", |
|
pm, i, pm->pm_pai[i].pai_asid); |
|
} |
|
#endif |
|
#else |
#else |
/* |
/* |
* Handle the case of an UP kernel which only has, at most, one TLB. |
* Handle the case of an UP kernel which only has, at most, one ASID. |
* If the pmap has an ASID allocated, free it. |
* If the pmap has an ASID allocated, free it. |
*/ |
*/ |
struct pmap_tlb_info * const ti = &pmap_tlb0_info; |
struct pmap_tlb_info * const ti = &pmap_tlb0_info; |
Line 1044 pmap_tlb_asid_release_all(struct pmap *p |
|
Line 996 pmap_tlb_asid_release_all(struct pmap *p |
|
if (curcpu()->ci_pmap_asid_cur == pai->pai_asid) { |
if (curcpu()->ci_pmap_asid_cur == pai->pai_asid) { |
tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); |
tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); |
} else { |
} else { |
pmap_tlb_pai_reset(ti, pai, pm); |
pmap_pai_reset(ti, pai, pm); |
} |
} |
} |
} |
TLBINFO_UNLOCK(ti); |
TLBINFO_UNLOCK(ti); |
#endif /* MULTIPROCESSOR */ |
#endif /* MULTIPROCESSOR */ |
UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); |
|
} |
} |
|
|
void |
void |