[BACK]Return to lock_stubs.S CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / arch / i386 / i386

Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.

Diff for /src/sys/arch/i386/i386/lock_stubs.S between version 1.2 and 1.2.8.7

version 1.2, 2007/02/09 21:55:04 version 1.2.8.7, 2007/12/03 19:03:28
Line 37 
Line 37 
  */   */
   
 /*  /*
  * Note on the 80386: the 80386 doesn't have a compare-and-exchange  
  * operation.  Stepping A of the i486 has these instructions wired to a  
  * different opcode, so should use these stubs also.  They are rare, so  
  * we don't make the effort.  
  *  
  * The sizes listed against each function are for a kernel compiled   * The sizes listed against each function are for a kernel compiled
  * with options MULTIPROCESSOR && DIAGNOSTIC && !I386_CPU.  The offsets   * with options MULTIPROCESSOR && DIAGNOSTIC.  Where possible we make
  * are for a kernel compiled without the I386_CPU option.  Where possible   * each routine fit into an assumed 64-byte cache line.  Please check
  * we make each routine fit into an assumed 64-byte cache line.   * alignment with 'objdump -d' after making changes.
  */   */
   
 #include "opt_multiprocessor.h"  #include "opt_multiprocessor.h"
 #include "opt_lockdebug.h"  #include "opt_lockdebug.h"
 #include "opt_cputype.h"  
 #include "opt_ddb.h"  #include "opt_ddb.h"
   
 #include <machine/asm.h>  #include <machine/asm.h>
Line 62 
Line 56 
 #define FULL  #define FULL
 #endif  #endif
   
 #if defined(I386_CPU)  
 #define STUB(name, alternate)                                   \  
         NENTRY(name) ;                                          \  
         cmpl    $CPUCLASS_386, _C_LABEL(cpu_class) ;            \  
         movl    4(%esp), %edx ;                                 \  
         je      _C_LABEL(alternate)  
 #define ALIGN64         .align  16      /* don't bother */  
 #define ALIGN32         .align  16      /* don't bother */  
 #else  
 #define STUB(name, alternate)                                   \  
         NENTRY(name) ;                                          \  
         movl    4(%esp), %edx  
 #define ALIGN64         .align  64  #define ALIGN64         .align  64
 #define ALIGN32         .align  32  #define ALIGN32         .align  32
 #endif  #define LOCK(name)      LABEL(name)     lock
   
 #if defined(MULTIPROCESSOR)  
 #define LOCK            lock  
 #else  
 #define LOCK            /* nothing */  
 #endif  
   
 #define END(name,a)     .align  a; LABEL(name)  #define END(name,a)     .align  a; LABEL(name)
   
 #if !defined(LOCKDEBUG)  #if !defined(LOCKDEBUG)
Line 95 
Line 70 
  */   */
         ALIGN64          ALIGN64
   
 STUB(mutex_enter, mutex_vector_enter)           /* 0x0000, 20 bytes */  ENTRY(mutex_enter)
           movl    4(%esp), %edx
         movl    CPUVAR(CURLWP), %ecx          movl    CPUVAR(CURLWP), %ecx
         xorl    %eax, %eax          xorl    %eax, %eax
         LOCK          LOCK(lockpatch1)
         cmpxchgl %ecx, MTX_OWNER(%edx)          cmpxchgl %ecx, MTX_OWNER(%edx)
         jnz,pn  _C_LABEL(mutex_vector_enter)          jnz,pn  _C_LABEL(mutex_vector_enter)
         ret          ret
Line 112  STUB(mutex_enter, mutex_vector_enter)  /
Line 88  STUB(mutex_enter, mutex_vector_enter)  /
  * on multiprocessor systems, and comments in arch/x86/include/lock.h about   * on multiprocessor systems, and comments in arch/x86/include/lock.h about
  * memory ordering on Intel x86 systems.   * memory ordering on Intel x86 systems.
  */   */
         ALIGN32  ENTRY(mutex_exit)
           movl    4(%esp), %edx
 STUB(mutex_exit, mutex_vector_exit)             /* 0x0020, 19 bytes */  
         movl    CPUVAR(CURLWP), %eax          movl    CPUVAR(CURLWP), %eax
         xorl    %ecx, %ecx          xorl    %ecx, %ecx
         cmpxchgl %ecx, MTX_OWNER(%edx)          cmpxchgl %ecx, MTX_OWNER(%edx)
Line 126  STUB(mutex_exit, mutex_vector_exit)  /* 
Line 101  STUB(mutex_exit, mutex_vector_exit)  /* 
  *   *
  * Acquire one hold on a RW lock.   * Acquire one hold on a RW lock.
  */   */
         ALIGN64  ENTRY(rw_enter)
           movl    4(%esp), %edx
 STUB(rw_enter, rw_vector_enter)                 /* 0x0040, 60 bytes */  
         cmpl    $RW_READER, 8(%esp)          cmpl    $RW_READER, 8(%esp)
         jne     2f          jne     2f
   
Line 139  STUB(rw_enter, rw_vector_enter)   /* 0x0
Line 113  STUB(rw_enter, rw_vector_enter)   /* 0x0
         testb   $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al          testb   $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
         leal    RW_READ_INCR(%eax), %ecx          leal    RW_READ_INCR(%eax), %ecx
         jnz,pn  _C_LABEL(rw_vector_enter)          jnz,pn  _C_LABEL(rw_vector_enter)
         LOCK          LOCK(lockpatch2)
         cmpxchgl %ecx, RW_OWNER(%edx)          cmpxchgl %ecx, RW_OWNER(%edx)
         jnz,pn  1b          jnz,pn  1b
         ret          ret
Line 150  STUB(rw_enter, rw_vector_enter)   /* 0x0
Line 124  STUB(rw_enter, rw_vector_enter)   /* 0x0
 2:      movl    CPUVAR(CURLWP), %ecx  2:      movl    CPUVAR(CURLWP), %ecx
         xorl    %eax, %eax          xorl    %eax, %eax
         orl     $RW_WRITE_LOCKED, %ecx          orl     $RW_WRITE_LOCKED, %ecx
         LOCK          LOCK(lockpatch3)
         cmpxchgl %ecx, RW_OWNER(%edx)          cmpxchgl %ecx, RW_OWNER(%edx)
         jnz,pn  _C_LABEL(rw_vector_enter)          jnz,pn  _C_LABEL(rw_vector_enter)
         ret          ret
Line 160  STUB(rw_enter, rw_vector_enter)   /* 0x0
Line 134  STUB(rw_enter, rw_vector_enter)   /* 0x0
  *   *
  * Release one hold on a RW lock.   * Release one hold on a RW lock.
  */   */
         ALIGN64  ENTRY(rw_exit)
           movl    4(%esp), %edx
 STUB(rw_exit, rw_vector_exit)                   /* 0x0080, 61 bytes */  
         movl    RW_OWNER(%edx), %eax          movl    RW_OWNER(%edx), %eax
         testb   $RW_WRITE_LOCKED, %al          testb   $RW_WRITE_LOCKED, %al
         jnz     2f          jnz     2f
Line 175  STUB(rw_exit, rw_vector_exit)   /* 0x008
Line 148  STUB(rw_exit, rw_vector_exit)   /* 0x008
         cmpl    $RW_READ_INCR, %eax          cmpl    $RW_READ_INCR, %eax
         leal    -RW_READ_INCR(%eax), %ecx          leal    -RW_READ_INCR(%eax), %ecx
         jb,pn   3f          jb,pn   3f
         LOCK          LOCK(lockpatch4)
         cmpxchgl %ecx, RW_OWNER(%edx)          cmpxchgl %ecx, RW_OWNER(%edx)
         jnz,pn  1b          jnz,pn  1b
         ret          ret
Line 186  STUB(rw_exit, rw_vector_exit)   /* 0x008
Line 159  STUB(rw_exit, rw_vector_exit)   /* 0x008
 2:      leal    -RW_WRITE_LOCKED(%eax), %ecx  2:      leal    -RW_WRITE_LOCKED(%eax), %ecx
         subl    CPUVAR(CURLWP), %ecx          subl    CPUVAR(CURLWP), %ecx
         jnz,pn  3f          jnz,pn  3f
         LOCK          LOCK(lockpatch5)
         cmpxchgl %ecx, RW_OWNER(%edx)          cmpxchgl %ecx, RW_OWNER(%edx)
         jnz,pn  3f          jnz,pn  3f
         ret          ret
Line 203  STUB(rw_exit, rw_vector_exit)   /* 0x008
Line 176  STUB(rw_exit, rw_vector_exit)   /* 0x008
  *   *
  * Acquire a spin mutex and post a load fence.   * Acquire a spin mutex and post a load fence.
  */   */
         ALIGN64  ENTRY(mutex_spin_enter)
           movl    4(%esp), %edx
 STUB(mutex_spin_enter, mutex_vector_enter)      /* 0x00c0, 51 bytes */          movl    CPUVAR(ILEVEL), %ecx
         movl    CPUVAR(SELF150), %eax          subl    $1, CPUVAR(MTX_COUNT)           /* decl does not set CF */
         movl    (CPU_INFO_ILEVEL-0x150)(%eax), %ecx  
         subl    $1, (CPU_INFO_MTX_COUNT-0x150)(%eax)/* decl does not set CF */  
         jnc     1f          jnc     1f
         movl    %ecx, (CPU_INFO_MTX_OLDSPL-0x150)(%eax)          movl    %ecx, CPUVAR(MTX_OLDSPL)
 1:      movb    MTX_IPL(%edx), %ch  1:      movb    MTX_IPL(%edx), %ch
         cmpb    %ch, %cl          cmpb    %ch, %cl
         jg,pn   2f          jg,pn   2f
         movb    %ch, (CPU_INFO_ILEVEL-0x150)(%eax)/* splraiseipl() */          movb    %ch, CPUVAR(ILEVEL)             /* splraiseipl() */
 2:  2:
 #if defined(FULL)  #ifdef FULL
         mov     $0x0100, %eax                   /* new + expected value */          mov     $0x0100, %eax                   /* new + expected value */
         LOCK          LOCK(lockpatch11)
         cmpxchgb %ah, MTX_LOCK(%edx)            /* lock it */          cmpxchgb %ah, MTX_LOCK(%edx)            /* lock it */
         jnz,pn  _C_LABEL(mutex_spin_retry)          jnz,pn  _C_LABEL(mutex_spin_retry)
 #endif  #endif
Line 228  STUB(mutex_spin_enter, mutex_vector_ente
Line 199  STUB(mutex_spin_enter, mutex_vector_ente
 LABEL(mutex_spin_enter_end)  LABEL(mutex_spin_enter_end)
   
 /*  /*
  * void mutex_spin_exit(kmutex_t *mtx);  
  *  
  * Release a spin mutex and post a store fence.   * Release a spin mutex and post a store fence.
  */   */
         ALIGN64  ENTRY(mutex_spin_exit)
           movl    4(%esp), %edx
 STUB(mutex_spin_exit, mutex_vector_exit)        /* 0x0100, 50 bytes */          movl    CPUVAR(MTX_OLDSPL), %ecx
 #if defined(DIAGNOSTIC)          incl    CPUVAR(MTX_COUNT)
         movl    $0x0001, %eax                   /* new + expected value */          movb    $0, MTX_LOCK(%edx)              /* zero */
         cmpxchgb %ah, MTX_LOCK(%edx)  
         jnz,pn  _C_LABEL(mutex_vector_exit)  
 #elif defined(MULTIPROCESSOR)  
         movb    $0x00,MTX_LOCK(%edx)  
 #endif  
         movl    CPUVAR(SELF150), %eax  
         movl    (CPU_INFO_MTX_OLDSPL-0x150)(%eax), %ecx  
         incl    (CPU_INFO_MTX_COUNT-0x150)(%eax)  
         jnz     1f          jnz     1f
         cmpl    (CPU_INFO_ILEVEL-0x150)(%eax), %ecx          movl    %fs:CPU_INFO_IUNMASK(,%ecx,4), %edx
         movl    %ecx, 4(%esp)  
         jae     1f  
         movl    (CPU_INFO_IUNMASK-0x150)(%eax,%ecx,4), %edx  
         cli          cli
         testl   (CPU_INFO_IPENDING-0x150)(%eax), %edx          testl   CPUVAR(IPENDING), %edx
           movl    %ecx, 4(%esp)
         jnz     _C_LABEL(Xspllower)             /* does sti */          jnz     _C_LABEL(Xspllower)             /* does sti */
         movl    %ecx, (CPU_INFO_ILEVEL-0x150)(%eax)          movl    %ecx, CPUVAR(ILEVEL)
         sti          sti
 1:      ret  1:      ret
           nop                                     /* XXX round up */
         ALIGN64          .align  32
 LABEL(mutex_spin_exit_end)  LABEL(mutex_spin_exit_end)
   
 #if !defined(I386_CPU) && defined(I686_CPU) && !defined(DIAGNOSTIC)  
   
 /*  /*
  * Patch for i686 CPUs where cli/sti is prohibitavely expensive.   * Patch for i686 CPUs where cli/sti is prohibitavely expensive.
  * Must be the same size as mutex_spin_exit().   * Must be the same size as mutex_spin_exit().
  */   */
         ALIGN64  ENTRY(i686_mutex_spin_exit)
   
 ENTRY(i686_mutex_spin_exit)                     /* 64 bytes */  
         mov     4(%esp),%edx          mov     4(%esp),%edx
         xorl    %eax,%eax          movl    CPUVAR(MTX_OLDSPL), %ecx
         pushl   %edi          incl    CPUVAR(MTX_COUNT)
         fs          movb    %ch, MTX_LOCK(%edx)             /* zero */
         movl    (CPU_INFO_SELF150)(%eax), %edi  /* now splx() */  
         pushl   %ebx  
         movl    (CPU_INFO_MTX_OLDSPL-0x150)(%edi), %ecx  
         incl    (CPU_INFO_MTX_COUNT-0x150)(%edi)  
         movb    %al, MTX_LOCK(%edx)             /* zero */  
         movl    (CPU_INFO_ILEVEL-0x150)(%edi), %edx  
         jnz     1f          jnz     1f
         cmpl    %edx, %ecx                      /* new level is lower? */          pushl   %ebx
         movl    (CPU_INFO_IPENDING-0x150)(%edi), %eax  0:
         jae,pn  1f          movl    CPUVAR(IPENDING), %eax
         testl   %eax,(CPU_INFO_IUNMASK-0x150)(%edi,%ecx,4)          testl   %eax, %fs:CPU_INFO_IUNMASK(,%ecx,4)
         movl    %eax, %ebx          movl    %eax, %ebx
         /*          /*
          * On a P4 this jump is cheaper than patching in junk using           * On a P4 this jump is cheaper than patching in junk using
          * cmovnz.  Is cmpxchg expensive if it fails?           * cmovnz.  Is cmpxchg expensive if it fails?
          */           */
         jnz,pn  2f          jnz     2f
         cmpxchg8b (CPU_INFO_ISTATE-0x150)(%edi) /* swap in new ilevel */          cmpxchg8b CPUVAR(ISTATE)                /* swap in new ilevel */
         jnz,pn  2f          jnz     0b
 1:  
         popl    %ebx          popl    %ebx
         popl    %edi  1:
         ret          ret
 2:  2:
         popl    %ebx          popl    %ebx
         popl    %edi  
         movl    %ecx,4(%esp)          movl    %ecx,4(%esp)
 LABEL(i686_mutex_spin_exit_patch)  LABEL(i686_mutex_spin_exit_patch)
         jmp     _C_LABEL(Xspllower)          jmp     _C_LABEL(Xspllower)
         ALIGN64          .align  32
 LABEL(i686_mutex_spin_exit_end)  LABEL(i686_mutex_spin_exit_end)
   
 #endif  /* !defined(I386_CPU) && defined(I686_CPU) && !defined(DIAGNOSTIC) */  
   
 #else   /* !__XEN__ */  #else   /* !__XEN__ */
   
 /* For now; strong alias not working for some reason. */  /* For now; strong alias not working for some reason. */
 NENTRY(mutex_spin_enter)  ENTRY(mutex_spin_enter)
         jmp     _C_LABEL(mutex_vector_enter)          jmp     _C_LABEL(mutex_vector_enter)
   
 NENTRY(mutex_spin_exit)  ENTRY(mutex_spin_exit)
         jmp     _C_LABEL(mutex_vector_exit)          jmp     _C_LABEL(mutex_vector_exit)
   
 #endif  /* !__XEN__ */  #endif  /* !__XEN__ */
Line 325  NENTRY(mutex_spin_exit)
Line 270  NENTRY(mutex_spin_exit)
  *   *
  * Perform an atomic compare-and-set operation.   * Perform an atomic compare-and-set operation.
  */   */
         ALIGN64  ENTRY(_lock_cas)
           mov     4(%esp),%edx
 STUB(_lock_cas, _80386_lock_cas)                /* 32 bytes */  
         movl    8(%esp), %eax          movl    8(%esp), %eax
         movl    12(%esp), %ecx          movl    12(%esp), %ecx
         LOCK          LOCK(lockpatch6)
         cmpxchgl %ecx, (%edx)          cmpxchgl %ecx, (%edx)
         movl    $0, %eax          movl    $0, %eax
         setz    %al          setz    %al
         ret          ret
   
 #ifdef I386_CPU  
 /*  
  * Since we can't do compare-and-exchange atomically with an 80386, we must  
  * disable interrupts in order to support preemption.  On the i386 this is  
  * cheap to do.  For other architectures a restartable sequence is usually  
  * a better option.  
  */  
 _80386_lock_cas:  
         movl    8(%esp), %eax  
         movl    12(%esp), %ecx  
         cli  
         cmpl    %eax, (%edx)  
         jne     1f  
         movl    %ecx, (%edx)  
         movb    $1, %al  
         sti  
         ret  
   
 1:      sti  
         xorl    %eax, %eax  
         ret  
 #endif  /* I386_CPU */  
   
 /*  /*
  * Memory barrier operations, may be patched at runtime.   * Memory barrier operations, may be patched at runtime.
  */   */
         .align  8          .align  8
   
 NENTRY(mb_read)  ENTRY(mb_read)
         lock          LOCK(lockpatch7)
         addl    $0, 0(%esp)          addl    $0, 0(%esp)
         ret          ret
 END(mb_read_end, 8)  END(mb_read_end, 8)
   
 NENTRY(mb_write)  ENTRY(mb_write)
         /* Nothing at the moment. */          nop
         ret          ret
 END(mb_write_end, 8)  END(mb_write_end, 8)
   
 NENTRY(mb_memory)  ENTRY(mb_memory)
         lock          LOCK(lockpatch8)
         addl    $0, 0(%esp)          addl    $0, 0(%esp)
         ret          ret
 END(mb_memory_end, 8)  END(mb_memory_end, 8)
   
 #ifdef I686_CPU  ENTRY(sse2_mb_read)
 NENTRY(sse2_mb_read)  
         lfence          lfence
         ret          ret
 END(sse2_mb_read_end, 8)  END(sse2_mb_read_end, 8)
   
 NENTRY(sse2_mb_memory)  ENTRY(sse2_mb_memory)
         mfence          mfence
         ret          ret
 END(sse2_mb_memory_end, 8)  END(sse2_mb_memory_end, 8)
 #endif  /* I686_CPU */  
   
 /*  /*
  * Make sure code after the ret is properly encoded with nopness   * Patchpoints to replace with NOP when ncpu == 1.
  * by gas, or could stall newer processors.  
  */   */
   #ifndef LOCKDEBUG
 NENTRY(x86_mb_nop)  LABEL(x86_lockpatch)
         ret          .long   lockpatch1, lockpatch2, lockpatch3, lockpatch4
 END(x86_mb_nop_end, 8)          .long   lockpatch5, lockpatch6, lockpatch7, lockpatch8
   #if defined(FULL) && !defined(__XEN__)
           .long   lockpatch11
   #endif
           .long   0
   #endif
Legend:
Removed from v.1.2  
changed lines
  Added in v.1.2.8.7
CVSweb <webmaster@jp.NetBSD.org>