[BACK]Return to pmap_tlb.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / uvm / pmap

Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.

Diff for /src/sys/uvm/pmap/pmap_tlb.c between version 1.10.2.5 and 1.11

version 1.10.2.5, 2017/08/28 17:53:18 version 1.11, 2015/04/18 16:58:31
Line 181  u_int pmap_ntlbs = 1;
Line 181  u_int pmap_ntlbs = 1;
 #define TLBINFO_ASID_INUSE_P(ti, asid) \  #define TLBINFO_ASID_INUSE_P(ti, asid) \
         __BITMAP_ISSET_P((ti)->ti_asid_bitmap, (asid))          __BITMAP_ISSET_P((ti)->ti_asid_bitmap, (asid))
   
   static void
   pmap_pai_check(struct pmap_tlb_info *ti)
   {
   #ifdef DIAGNOSTIC
           struct pmap_asid_info *pai;
           LIST_FOREACH(pai, &ti->ti_pais, pai_link) {
                   KASSERT(pai != NULL);
                   KASSERT(PAI_PMAP(pai, ti) != pmap_kernel());
                   KASSERT(pai->pai_asid > KERNEL_PID);
                   KASSERT(TLBINFO_ASID_INUSE_P(ti, pai->pai_asid));
           }
   #endif
   }
   
 #ifdef MULTIPROCESSOR  #ifdef MULTIPROCESSOR
 __unused static inline bool  __unused static inline bool
 pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti)  pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti)
Line 203  pmap_tlb_intersecting_onproc_p(pmap_t pm
Line 217  pmap_tlb_intersecting_onproc_p(pmap_t pm
 }  }
 #endif  #endif
   
 static void  static inline void
 pmap_tlb_pai_check(struct pmap_tlb_info *ti, bool locked_p)  pmap_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai,
 {  
 #ifdef DIAGNOSTIC  
         struct pmap_asid_info *pai;  
         if (!locked_p)  
                 TLBINFO_LOCK(ti);  
         LIST_FOREACH(pai, &ti->ti_pais, pai_link) {  
                 KASSERT(pai != NULL);  
                 KASSERT(PAI_PMAP(pai, ti) != pmap_kernel());  
                 KASSERT(pai->pai_asid > KERNEL_PID);  
                 KASSERTMSG(pai->pai_asid <= ti->ti_asid_max,  
                     "pm %p asid %#x", PAI_PMAP(pai, ti), pai->pai_asid);  
                 KASSERTMSG(TLBINFO_ASID_INUSE_P(ti, pai->pai_asid),  
                     "pm %p asid %u", PAI_PMAP(pai, ti), pai->pai_asid);  
 #ifdef MULTIPROCESSOR  
                 KASSERT(pmap_tlb_intersecting_active_p(PAI_PMAP(pai, ti), ti));  
 #endif  
         }  
         if (!locked_p)  
                 TLBINFO_UNLOCK(ti);  
 #endif  
 }  
   
 static void  
 pmap_tlb_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai,  
         struct pmap *pm)          struct pmap *pm)
 {  {
         UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);  
         UVMHIST_LOG(maphist, "(ti=%p, pai=%p, pm=%p): asid %u",  
             ti, pai, pm, pai->pai_asid);  
   
         /*          /*
          * We must have an ASID but it must not be onproc (on a processor).           * We must have an ASID but it must not be onproc (on a processor).
          */           */
         KASSERT(pai->pai_asid > KERNEL_PID);          KASSERT(pai->pai_asid > KERNEL_PID);
         KASSERT(pai->pai_asid <= ti->ti_asid_max);  
 #if defined(MULTIPROCESSOR)  #if defined(MULTIPROCESSOR)
         KASSERT(pmap_tlb_intersecting_active_p(pm, ti));  
         KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));          KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
 #endif  #endif
         LIST_REMOVE(pai, pai_link);          LIST_REMOVE(pai, pai_link);
Line 285  pmap_tlb_pai_reset(struct pmap_tlb_info 
Line 269  pmap_tlb_pai_reset(struct pmap_tlb_info 
 #if PMAP_TLB_MAX == 1  #if PMAP_TLB_MAX == 1
         kcpuset_zero(pm->pm_active);          kcpuset_zero(pm->pm_active);
 #else  #else
         kcpuset_remove(pm->pm_active, ti->ti_kcpuset);          kcpuset_atomicly_remove(pm->pm_active, ti->ti_kcpuset);
 #endif  #endif
         KASSERT(!pmap_tlb_intersecting_active_p(pm, ti));  
 #endif /* MULTIPROCESSOR */  #endif /* MULTIPROCESSOR */
   
         UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);  
 }  }
   
 void  void
Line 338  pmap_tlb_info_init(struct pmap_tlb_info 
Line 319  pmap_tlb_info_init(struct pmap_tlb_info 
                 ti->ti_asid_hint = KERNEL_PID + 1;                  ti->ti_asid_hint = KERNEL_PID + 1;
                 ti->ti_asid_max = pmap_tlbs[0]->ti_asid_max;                  ti->ti_asid_max = pmap_tlbs[0]->ti_asid_max;
                 ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID;                  ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID;
                 ti->ti_tlbinvop = TLBINV_NOBODY;                  ti->ti_tlbinvop = TLBINV_NOBODY,
                 ti->ti_victim = NULL;                  ti->ti_victim = NULL;
                 kcpuset_create(&ti->ti_kcpuset, true);                  kcpuset_create(&ti->ti_kcpuset, true);
                 ti->ti_index = pmap_ntlbs++;                  ti->ti_index = pmap_ntlbs++;
Line 413  pmap_tlb_asid_reinitialize(struct pmap_t
Line 394  pmap_tlb_asid_reinitialize(struct pmap_t
         const size_t asid_bitmap_words =          const size_t asid_bitmap_words =
             ti->ti_asid_max / (8 * sizeof(ti->ti_asid_bitmap[0]));              ti->ti_asid_max / (8 * sizeof(ti->ti_asid_bitmap[0]));
   
         UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);          pmap_pai_check(ti);
         UVMHIST_LOG(maphist, "(ti=%p, op=%u)", ti, op, 0, 0);  
   
         pmap_tlb_pai_check(ti, true);  
   
         ti->ti_evcnt_asid_reinits.ev_count++;          ti->ti_evcnt_asid_reinits.ev_count++;
   
Line 432  pmap_tlb_asid_reinitialize(struct pmap_t
Line 410  pmap_tlb_asid_reinitialize(struct pmap_t
         }          }
   
         switch (op) {          switch (op) {
 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)  #if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN)
         case TLBINV_ALL:          case TLBINV_ALL:
                 tlb_invalidate_all();                  tlb_invalidate_all();
                 break;                  break;
         case TLBINV_ALLUSER:          case TLBINV_ALLUSER:
                 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max);                  tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max);
                 break;                  break;
 #endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */  #endif /* MULTIPROCESSOR && PMAP_NEED_TLB_SHOOTDOWN */
         case TLBINV_NOBODY: {          case TLBINV_NOBODY: {
                 /*                  /*
                  * If we are just reclaiming ASIDs in the TLB, let's go find                   * If we are just reclaiming ASIDs in the TLB, let's go find
Line 450  pmap_tlb_asid_reinitialize(struct pmap_t
Line 428  pmap_tlb_asid_reinitialize(struct pmap_t
                  * and clear the ASID bitmap.  That will force everyone to                   * and clear the ASID bitmap.  That will force everyone to
                  * allocate a new ASID.                   * allocate a new ASID.
                  */                   */
 #if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN)  #if !defined(MULTIPROCESSOR) || defined(PMAP_NEED_TLB_SHOOTDOWN)
                 pmap_tlb_asid_check();                  pmap_tlb_asid_check();
                 const u_int asids_found = tlb_record_asids(ti->ti_asid_bitmap,                  const u_int asids_found = tlb_record_asids(ti->ti_asid_bitmap);
                     ti->ti_asid_max);  
                 pmap_tlb_asid_check();                  pmap_tlb_asid_check();
                 KASSERT(asids_found == pmap_tlb_asid_count(ti));                  KASSERT(asids_found == pmap_tlb_asid_count(ti));
                 if (__predict_false(asids_found >= ti->ti_asid_max / 2)) {                  if (__predict_false(asids_found >= ti->ti_asid_max / 2)) {
                         tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max);                          tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max);
 #else /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */  #else /* MULTIPROCESSOR && !PMAP_NEED_TLB_SHOOTDOWN */
                         /*                          /*
                          * For those systems (PowerPC) that don't require                           * For those systems (PowerPC) that don't require
                          * cross cpu TLB shootdowns, we have to invalidate the                           * cross cpu TLB shootdowns, we have to invalidate the
Line 469  pmap_tlb_asid_reinitialize(struct pmap_t
Line 446  pmap_tlb_asid_reinitialize(struct pmap_t
                          * nightmare).                           * nightmare).
                          */                           */
                         tlb_invalidate_all();                          tlb_invalidate_all();
 #endif /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */  #endif /* MULTIPROCESSOR && !PMAP_NEED_TLB_SHOOTDOWN */
                         ti->ti_asid_bitmap[0] = (2 << KERNEL_PID) - 1;                          ti->ti_asid_bitmap[0] = (2 << KERNEL_PID) - 1;
                         for (size_t word = 1;                          for (size_t word = 1;
                              word <= asid_bitmap_words;                               word <= asid_bitmap_words;
Line 477  pmap_tlb_asid_reinitialize(struct pmap_t
Line 454  pmap_tlb_asid_reinitialize(struct pmap_t
                                 ti->ti_asid_bitmap[word] = 0;                                  ti->ti_asid_bitmap[word] = 0;
                         }                          }
                         ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID;                          ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID;
 #if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN)  #if !defined(MULTIPROCESSOR) || defined(PMAP_NEED_TLB_SHOOTDOWN)
                 } else {                  } else {
                         ti->ti_asids_free -= asids_found;                          ti->ti_asids_free -= asids_found;
                 }                  }
 #endif /* !MULTIPROCESSOR || PMAP_TLB_NEED_SHOOTDOWN */  #endif /* !MULTIPROCESSOR || PMAP_NEED_TLB_SHOOTDOWN */
                 KASSERTMSG(ti->ti_asids_free <= ti->ti_asid_max, "%u",                  KASSERTMSG(ti->ti_asids_free <= ti->ti_asid_max, "%u",
                     ti->ti_asids_free);                      ti->ti_asids_free);
                 break;                  break;
Line 513  pmap_tlb_asid_reinitialize(struct pmap_t
Line 490  pmap_tlb_asid_reinitialize(struct pmap_t
                 if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {                  if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
                         KASSERT(op == TLBINV_NOBODY);                          KASSERT(op == TLBINV_NOBODY);
                 } else {                  } else {
                         pmap_tlb_pai_reset(ti, pai, pm);                          pmap_pai_reset(ti, pai, pm);
                 }                  }
         }          }
 #ifdef DIAGNOSTIC  #ifdef DIAGNOSTIC
Line 521  pmap_tlb_asid_reinitialize(struct pmap_t
Line 498  pmap_tlb_asid_reinitialize(struct pmap_t
         KASSERTMSG(free_count == ti->ti_asids_free,          KASSERTMSG(free_count == ti->ti_asids_free,
             "bitmap error: %zu != %u", free_count, ti->ti_asids_free);              "bitmap error: %zu != %u", free_count, ti->ti_asids_free);
 #endif  #endif
         UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);  
 }  }
   
 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)  #if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN)
 #if PMAP_TLB_MAX == 1  #if PMAP_MAX_TLB == 1
 #error shootdown not required for single TLB systems  #error shootdown not required for single TLB systems
 #endif  #endif
 void  void
Line 551  pmap_tlb_shootdown_process(void)
Line 527  pmap_tlb_shootdown_process(void)
                  */                   */
                 struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti);                  struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti);
                 KASSERT(ti->ti_victim != pmap_kernel());                  KASSERT(ti->ti_victim != pmap_kernel());
                 if (!pmap_tlb_intersecting_onproc_p(ti->ti_victim, ti)) {                  if (!pmap_tlb_intersecting_onproc_p(ti_victim->pm_onproc, ti)) {
                         /*                          /*
                          * The victim is an active pmap so we will just                           * The victim is an active pmap so we will just
                          * invalidate its TLB entries.                           * invalidate its TLB entries.
Line 568  pmap_tlb_shootdown_process(void)
Line 544  pmap_tlb_shootdown_process(void)
                          * ASID.                           * ASID.
                          */                           */
                         KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));                          KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
                         pmap_tlb_pai_reset(ti, pai, PAI_PMAP(pai, ti));                          pmap_pai_reset(ti, pai, PAI_PMAP(pai, ti));
                 }                  }
                 break;                  break;
         }          }
Line 632  bool
Line 608  bool
 pmap_tlb_shootdown_bystanders(pmap_t pm)  pmap_tlb_shootdown_bystanders(pmap_t pm)
 {  {
         /*          /*
          * We don't need to deal with our own TLB.           * We don't need to deal our own TLB.
          */           */
   
         UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);  
   
         kcpuset_t *pm_active;          kcpuset_t *pm_active;
   
         kcpuset_clone(&pm_active, pm->pm_active);          kcpuset_clone(&pm_active, pm->pm_active);
         kcpuset_remove(pm_active, cpu_tlb_info(curcpu())->ti_kcpuset);          kcpuset_atomicly_remove(pm->pm_active,
               cpu_tlb_info(curcpu())->ti_kcpuset);
         const bool kernel_p = (pm == pmap_kernel());          const bool kernel_p = (pm == pmap_kernel());
         bool ipi_sent = false;          bool ipi_sent = false;
   
Line 660  pmap_tlb_shootdown_bystanders(pmap_t pm)
Line 635  pmap_tlb_shootdown_bystanders(pmap_t pm)
                 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);                  struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
                 kcpuset_remove(pm_active, ti->ti_kcpuset);                  kcpuset_remove(pm_active, ti->ti_kcpuset);
                 TLBINFO_LOCK(ti);                  TLBINFO_LOCK(ti);
                 cpuid_t j = kcpuset_ffs_intersecting(pm->pm_onproc,                  if (pmap_tlb_intersecting_onproc_p(pm, ti)) {
                     ti->ti_kcpuset);                          cpuid_t j = kcpuset_ffs_intersecting(pm->pm_onproc,
                 // post decrement since ffs returns bit + 1 or 0 if no bit                              ti->ti_kcpuset);
                 if (j-- > 0) {  
                         if (kernel_p) {                          if (kernel_p) {
                                 ti->ti_tlbinvop =                                  ti->ti_tlbinvop =
                                     TLBINV_KERNEL_MAP(ti->ti_tlbinvop);                                      TLBINV_KERNEL_MAP(ti->ti_tlbinvop);
Line 707  pmap_tlb_shootdown_bystanders(pmap_t pm)
Line 681  pmap_tlb_shootdown_bystanders(pmap_t pm)
                          * And best of all, we avoid an IPI.                           * And best of all, we avoid an IPI.
                          */                           */
                         KASSERT(!kernel_p);                          KASSERT(!kernel_p);
                         pmap_tlb_pai_reset(ti, pai, pm);                          pmap_pai_reset(ti, pai, pm);
                         //ti->ti_evcnt_lazy_shots.ev_count++;                          //ti->ti_evcnt_lazy_shots.ev_count++;
                 }                  }
                 TLBINFO_UNLOCK(ti);                  TLBINFO_UNLOCK(ti);
Line 715  pmap_tlb_shootdown_bystanders(pmap_t pm)
Line 689  pmap_tlb_shootdown_bystanders(pmap_t pm)
   
         kcpuset_destroy(pm_active);          kcpuset_destroy(pm_active);
   
         UVMHIST_LOG(maphist, " <-- done (ipi_sent=%d)", ipi_sent, 0, 0, 0);  
   
         return ipi_sent;          return ipi_sent;
 }  }
 #endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */  #endif /* MULTIPROCESSOR && PMAP_NEED_TLB_SHOOTDOWN */
   
 #ifndef PMAP_HWPAGEWALKER  #ifndef PMAP_TLB_HWPAGEWALKER
 int  int
 pmap_tlb_update_addr(pmap_t pm, vaddr_t va, pt_entry_t pte, u_int flags)  pmap_tlb_update_addr(pmap_t pm, vaddr_t va, pt_entry_t pt_entry, u_int flags)
 {  {
         struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());          struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
         struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);          struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
         int rv = -1;          int rv = -1;
   
         UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);  
         UVMHIST_LOG(maphist,  
             " (pm=%p va=%#"PRIxVADDR", pte=%#"PRIxPTE" flags=%#x)",  
             pm, va, pte_value(pte), flags);  
   
         KASSERT(kpreempt_disabled());          KASSERT(kpreempt_disabled());
   
         KASSERTMSG(pte_valid_p(pte), "va %#"PRIxVADDR" %#"PRIxPTE,  
             va, pte_value(pte));  
   
         TLBINFO_LOCK(ti);          TLBINFO_LOCK(ti);
         if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {          if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {
                 pmap_tlb_asid_check();                  pmap_tlb_asid_check();
                 rv = tlb_update_addr(va, pai->pai_asid, pte,                  rv = tlb_update_addr(va, pai->pai_asid, pt_entry,
                     (flags & PMAP_TLB_INSERT) != 0);                      (flags & PMAP_TLB_INSERT) != 0);
                 pmap_tlb_asid_check();                  pmap_tlb_asid_check();
                 UVMHIST_LOG(maphist,          }
                      "   %d <-- tlb_update_addr(%#"PRIxVADDR", %#x, %#"PRIxPTE", ...)",  #if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN)
                      rv, va, pai->pai_asid, pte_value(pte));          pm->pm_shootdown_pending = (flags & PMAP_TLB_NEED_IPI) != 0;
                 KASSERTMSG((flags & PMAP_TLB_INSERT) == 0 || rv == 1,  
                     "pmap %p (asid %u) va %#"PRIxVADDR" pte %#"PRIxPTE" rv %d",  
                     pm, pai->pai_asid, va, pte_value(pte), rv);  
         }  
 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)  
         if (flags & PMAP_TLB_NEED_IPI)  
                 pm->pm_shootdown_pending = 1;  
 #endif  #endif
         TLBINFO_UNLOCK(ti);          TLBINFO_UNLOCK(ti);
   
         UVMHIST_LOG(maphist, "   <-- done (rv=%d)", rv, 0, 0, 0);  
   
         return rv;          return rv;
 }  }
 #endif /* !PMAP_HWPAGEWALKER */  #endif /* !PMAP_TLB_HWPAGEWALKER */
   
 void  void
 pmap_tlb_invalidate_addr(pmap_t pm, vaddr_t va)  pmap_tlb_invalidate_addr(pmap_t pm, vaddr_t va)
Line 771  pmap_tlb_invalidate_addr(pmap_t pm, vadd
Line 726  pmap_tlb_invalidate_addr(pmap_t pm, vadd
         struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);          struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
   
         UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);          UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
         UVMHIST_LOG(maphist, " (pm=%p va=%#"PRIxVADDR") ti=%p asid=%#x",  
             pm, va, ti, pai->pai_asid);  
   
         KASSERT(kpreempt_disabled());          KASSERT(kpreempt_disabled());
   
           UVMHIST_LOG(maphist, " (pm=%#x va=%#x) ti=%#x asid=%#x",
               pm, va, ti, pai->pai_asid);
   
         TLBINFO_LOCK(ti);          TLBINFO_LOCK(ti);
         if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {          if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {
                 pmap_tlb_asid_check();                  pmap_tlb_asid_check();
                 UVMHIST_LOG(maphist, " invalidating %#"PRIxVADDR" asid %#x",                  UVMHIST_LOG(maphist, " invalidating %#x asid %#x",
                     va, pai->pai_asid, 0, 0);                      va, pai->pai_asid, 0, 0);
                 tlb_invalidate_addr(va, pai->pai_asid);                  tlb_invalidate_addr(va, pai->pai_asid);
                 pmap_tlb_asid_check();                  pmap_tlb_asid_check();
         }          }
 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)  #if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN)
         pm->pm_shootdown_pending = 1;          pm->pm_shootdown_pending = 1;
 #endif  #endif
         TLBINFO_UNLOCK(ti);          TLBINFO_UNLOCK(ti);
Line 826  pmap_tlb_asid_alloc(struct pmap_tlb_info
Line 782  pmap_tlb_asid_alloc(struct pmap_tlb_info
          * a new one.           * a new one.
          */           */
         if (__predict_true(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) {          if (__predict_true(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) {
                 const size_t nbpw = 8 * sizeof(ti->ti_asid_bitmap[0]);                  const size_t nbpw __diagused = 8*sizeof(ti->ti_asid_bitmap[0]);
                 size_t i;                  size_t i;
                 u_long bits;                  u_long bits;
                 for (i = 0; (bits = ~ti->ti_asid_bitmap[i]) == 0; i++) {                  for (i = 0; (bits = ~ti->ti_asid_bitmap[i]) == 0; i++) {
Line 879  pmap_tlb_asid_alloc(struct pmap_tlb_info
Line 835  pmap_tlb_asid_alloc(struct pmap_tlb_info
 #if PMAP_TLB_MAX == 1  #if PMAP_TLB_MAX == 1
         kcpuset_copy(pm->pm_active, kcpuset_running);          kcpuset_copy(pm->pm_active, kcpuset_running);
 #else  #else
         kcpuset_merge(pm->pm_active, ti->ti_kcpuset);          kcpuset_atomicly_merge(pm->pm_active, ti->ti_kcpuset);
 #endif  #endif
 #endif  #endif
 }  }
Line 895  pmap_tlb_asid_acquire(pmap_t pm, struct 
Line 851  pmap_tlb_asid_acquire(pmap_t pm, struct 
         struct pmap_tlb_info * const ti = cpu_tlb_info(ci);          struct pmap_tlb_info * const ti = cpu_tlb_info(ci);
         struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);          struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
   
         UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);  
         UVMHIST_LOG(maphist, "(pm=%p, l=%p, ti=%p)", pm, l, ti, 0);  
   
         KASSERT(kpreempt_disabled());          KASSERT(kpreempt_disabled());
   
           UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
   
         /*          /*
          * Kernels use a fixed ASID and thus doesn't need to acquire one.           * Kernels use a fixed ASID and thus doesn't need to acquire one.
          */           */
Line 908  pmap_tlb_asid_acquire(pmap_t pm, struct 
Line 863  pmap_tlb_asid_acquire(pmap_t pm, struct 
                 return;                  return;
         }          }
   
           UVMHIST_LOG(maphist, " (pm=%#x, l=%#x, ti=%#x)", pm, l, ti, 0);
         TLBINFO_LOCK(ti);          TLBINFO_LOCK(ti);
         KASSERT(pai->pai_asid <= KERNEL_PID || pai->pai_link.le_prev != NULL);          KASSERT(pai->pai_asid <= KERNEL_PID || pai->pai_link.le_prev != NULL);
         KASSERT(pai->pai_asid > KERNEL_PID || pai->pai_link.le_prev == NULL);          KASSERT(pai->pai_asid > KERNEL_PID || pai->pai_link.le_prev == NULL);
         pmap_tlb_pai_check(ti, true);          pmap_pai_check(ti);
         if (__predict_false(!PMAP_PAI_ASIDVALID_P(pai, ti))) {          if (__predict_false(!PMAP_PAI_ASIDVALID_P(pai, ti))) {
                 /*                  /*
                  * If we've run out ASIDs, reinitialize the ASID space.                   * If we've run out ASIDs, reinitialize the ASID space.
Line 929  pmap_tlb_asid_acquire(pmap_t pm, struct 
Line 885  pmap_tlb_asid_acquire(pmap_t pm, struct 
                 pmap_tlb_asid_alloc(ti, pm, pai);                  pmap_tlb_asid_alloc(ti, pm, pai);
                 UVMHIST_LOG(maphist, "allocated asid %#x", pai->pai_asid, 0, 0, 0);                  UVMHIST_LOG(maphist, "allocated asid %#x", pai->pai_asid, 0, 0, 0);
         }          }
         pmap_tlb_pai_check(ti, true);  
 #if defined(MULTIPROCESSOR)  
         KASSERT(kcpuset_isset(pm->pm_active, cpu_index(ci)));  
 #endif  
   
         if (l == curlwp) {          if (l == curlwp) {
 #if defined(MULTIPROCESSOR)  #if defined(MULTIPROCESSOR)
Line 959  void
Line 911  void
 pmap_tlb_asid_deactivate(pmap_t pm)  pmap_tlb_asid_deactivate(pmap_t pm)
 {  {
         UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);          UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
   
         KASSERT(kpreempt_disabled());          KASSERT(kpreempt_disabled());
 #if defined(MULTIPROCESSOR)  #if defined(MULTIPROCESSOR)
         /*          /*
Line 982  pmap_tlb_asid_deactivate(pmap_t pm)
Line 933  pmap_tlb_asid_deactivate(pmap_t pm)
                 kcpuset_atomic_clear(pm->pm_onproc, cpu_index(ci));                  kcpuset_atomic_clear(pm->pm_onproc, cpu_index(ci));
         }          }
 #endif  #endif
         curcpu()->ci_pmap_asid_cur = KERNEL_PID;          curcpu()->ci_pmap_asid_cur = 0;
         UVMHIST_LOG(maphist, " <-- done (pm=%p)", pm, 0, 0, 0);          UVMHIST_LOG(maphist, " <-- done (pm=%#x)", pm, 0, 0, 0);
         tlb_set_asid(KERNEL_PID);          tlb_set_asid(KERNEL_PID);
         pmap_tlb_pai_check(cpu_tlb_info(curcpu()), false);  
 #if defined(DEBUG)  #if defined(DEBUG)
         pmap_tlb_asid_check();          pmap_tlb_asid_check();
 #endif  #endif
Line 994  pmap_tlb_asid_deactivate(pmap_t pm)
Line 944  pmap_tlb_asid_deactivate(pmap_t pm)
 void  void
 pmap_tlb_asid_release_all(struct pmap *pm)  pmap_tlb_asid_release_all(struct pmap *pm)
 {  {
         UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);  
         UVMHIST_LOG(maphist, "(pm=%p)", pm, 0, 0, 0);  
   
         KASSERT(pm != pmap_kernel());          KASSERT(pm != pmap_kernel());
 #if defined(MULTIPROCESSOR)  #if defined(MULTIPROCESSOR)
         //KASSERT(!kcpuset_iszero(pm->pm_onproc)); // XXX          //KASSERT(!kcpuset_iszero(pm->pm_onproc)); // XXX
         struct cpu_info * const ci __diagused = curcpu();  
         KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci)));  
 #if PMAP_TLB_MAX > 1  #if PMAP_TLB_MAX > 1
           struct cpu_info * const ci __diagused = curcpu();
         for (u_int i = 0; !kcpuset_iszero(pm->pm_active); i++) {          for (u_int i = 0; !kcpuset_iszero(pm->pm_active); i++) {
                 KASSERT(i < pmap_ntlbs);                  KASSERT(i < pmap_ntlbs);
                 struct pmap_tlb_info * const ti = pmap_tlbs[i];                  struct pmap_tlb_info * const ti = pmap_tlbs[i];
Line 1013  pmap_tlb_asid_release_all(struct pmap *p
Line 959  pmap_tlb_asid_release_all(struct pmap *p
                 TLBINFO_LOCK(ti);                  TLBINFO_LOCK(ti);
                 if (PMAP_PAI_ASIDVALID_P(pai, ti)) {                  if (PMAP_PAI_ASIDVALID_P(pai, ti)) {
                         /*                          /*
                          * This pmap should not be in use by any other cpu so                           * If this pmap isn't onproc on any of the cpus
                          * we can just reset and be happy.                           * belonging to this tlb domain, we can just reset
                            * the ASID and be done.
                          */                           */
                         if (ti->ti_victim == pm)                          if (!pmap_tlb_intersecting_onproc_p(pm, ti)) {
                                 ti->ti_victim = NULL;                                  KASSERT(ti->ti_victim != pm);
                         pmap_tlb_pai_reset(ti, pai, pm);                                  pmap_pai_reset(ti, pai, pm);
   #if PMAP_TLB_MAX == 1
                           } else {
                                   KASSERT(cpu_tlb_info(ci) == ti);
                                   tlb_invalidate_asids(pai->pai_asid,
                                       pai->pai_asid);
   #else
                           } else if (cpu_tlb_info(ci) == ti) {
                                   tlb_invalidate_asids(pai->pai_asid,
                                       pai->pai_asid);
                           } else {
                                   pm->pm_shootdown_needed = 1;
   #endif
                           }
                 }                  }
                 KASSERT(pai->pai_link.le_prev == NULL);  
                 TLBINFO_UNLOCK(ti);                  TLBINFO_UNLOCK(ti);
 #if PMAP_TLB_MAX > 1  #if PMAP_TLB_MAX > 1
         }          }
 #endif  #endif
 #ifdef DIAGNOSTIC  
         for (size_t i = 0; i < (PMAP_TLB_MAX > 1 ? pmap_ntlbs : 1); i++) {  
                 KASSERTMSG(pm->pm_pai[i].pai_asid == 0,  
                     "pm %p i %zu asid %u",  
                     pm, i, pm->pm_pai[i].pai_asid);  
         }  
 #endif  
 #else  #else
         /*          /*
          * Handle the case of an UP kernel which only has, at most, one TLB.           * Handle the case of an UP kernel which only has, at most, one ASID.
          * If the pmap has an ASID allocated, free it.           * If the pmap has an ASID allocated, free it.
          */           */
         struct pmap_tlb_info * const ti = &pmap_tlb0_info;          struct pmap_tlb_info * const ti = &pmap_tlb0_info;
Line 1044  pmap_tlb_asid_release_all(struct pmap *p
Line 996  pmap_tlb_asid_release_all(struct pmap *p
                 if (curcpu()->ci_pmap_asid_cur == pai->pai_asid) {                  if (curcpu()->ci_pmap_asid_cur == pai->pai_asid) {
                         tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);                          tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
                 } else {                  } else {
                         pmap_tlb_pai_reset(ti, pai, pm);                          pmap_pai_reset(ti, pai, pm);
                 }                  }
         }          }
         TLBINFO_UNLOCK(ti);          TLBINFO_UNLOCK(ti);
 #endif /* MULTIPROCESSOR */  #endif /* MULTIPROCESSOR */
         UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);  
 }  }
   
 void  void

Legend:
Removed from v.1.10.2.5  
changed lines
  Added in v.1.11

CVSweb <webmaster@jp.NetBSD.org>