version 1.2, 2007/02/09 21:55:04 |
version 1.2.8.7, 2007/12/03 19:03:28 |
|
|
*/ |
*/ |
|
|
/* |
/* |
* Note on the 80386: the 80386 doesn't have a compare-and-exchange |
|
* operation. Stepping A of the i486 has these instructions wired to a |
|
* different opcode, so should use these stubs also. They are rare, so |
|
* we don't make the effort. |
|
* |
|
* The sizes listed against each function are for a kernel compiled |
* The sizes listed against each function are for a kernel compiled |
* with options MULTIPROCESSOR && DIAGNOSTIC && !I386_CPU. The offsets |
* with options MULTIPROCESSOR && DIAGNOSTIC. Where possible we make |
* are for a kernel compiled without the I386_CPU option. Where possible |
* each routine fit into an assumed 64-byte cache line. Please check |
* we make each routine fit into an assumed 64-byte cache line. |
* alignment with 'objdump -d' after making changes. |
*/ |
*/ |
|
|
#include "opt_multiprocessor.h" |
#include "opt_multiprocessor.h" |
#include "opt_lockdebug.h" |
#include "opt_lockdebug.h" |
#include "opt_cputype.h" |
|
#include "opt_ddb.h" |
#include "opt_ddb.h" |
|
|
#include <machine/asm.h> |
#include <machine/asm.h> |
|
|
#define FULL |
#define FULL |
#endif |
#endif |
|
|
#if defined(I386_CPU) |
|
#define STUB(name, alternate) \ |
|
NENTRY(name) ; \ |
|
cmpl $CPUCLASS_386, _C_LABEL(cpu_class) ; \ |
|
movl 4(%esp), %edx ; \ |
|
je _C_LABEL(alternate) |
|
#define ALIGN64 .align 16 /* don't bother */ |
|
#define ALIGN32 .align 16 /* don't bother */ |
|
#else |
|
#define STUB(name, alternate) \ |
|
NENTRY(name) ; \ |
|
movl 4(%esp), %edx |
|
#define ALIGN64 .align 64 |
#define ALIGN64 .align 64 |
#define ALIGN32 .align 32 |
#define ALIGN32 .align 32 |
#endif |
#define LOCK(name) LABEL(name) lock |
|
|
#if defined(MULTIPROCESSOR) |
|
#define LOCK lock |
|
#else |
|
#define LOCK /* nothing */ |
|
#endif |
|
|
|
#define END(name,a) .align a; LABEL(name) |
#define END(name,a) .align a; LABEL(name) |
|
|
#if !defined(LOCKDEBUG) |
#if !defined(LOCKDEBUG) |
|
|
*/ |
*/ |
ALIGN64 |
ALIGN64 |
|
|
STUB(mutex_enter, mutex_vector_enter) /* 0x0000, 20 bytes */ |
ENTRY(mutex_enter) |
|
movl 4(%esp), %edx |
movl CPUVAR(CURLWP), %ecx |
movl CPUVAR(CURLWP), %ecx |
xorl %eax, %eax |
xorl %eax, %eax |
LOCK |
LOCK(lockpatch1) |
cmpxchgl %ecx, MTX_OWNER(%edx) |
cmpxchgl %ecx, MTX_OWNER(%edx) |
jnz,pn _C_LABEL(mutex_vector_enter) |
jnz,pn _C_LABEL(mutex_vector_enter) |
ret |
ret |
Line 112 STUB(mutex_enter, mutex_vector_enter) / |
|
Line 88 STUB(mutex_enter, mutex_vector_enter) / |
|
* on multiprocessor systems, and comments in arch/x86/include/lock.h about |
* on multiprocessor systems, and comments in arch/x86/include/lock.h about |
* memory ordering on Intel x86 systems. |
* memory ordering on Intel x86 systems. |
*/ |
*/ |
ALIGN32 |
ENTRY(mutex_exit) |
|
movl 4(%esp), %edx |
STUB(mutex_exit, mutex_vector_exit) /* 0x0020, 19 bytes */ |
|
movl CPUVAR(CURLWP), %eax |
movl CPUVAR(CURLWP), %eax |
xorl %ecx, %ecx |
xorl %ecx, %ecx |
cmpxchgl %ecx, MTX_OWNER(%edx) |
cmpxchgl %ecx, MTX_OWNER(%edx) |
Line 126 STUB(mutex_exit, mutex_vector_exit) /* |
|
Line 101 STUB(mutex_exit, mutex_vector_exit) /* |
|
* |
* |
* Acquire one hold on a RW lock. |
* Acquire one hold on a RW lock. |
*/ |
*/ |
ALIGN64 |
ENTRY(rw_enter) |
|
movl 4(%esp), %edx |
STUB(rw_enter, rw_vector_enter) /* 0x0040, 60 bytes */ |
|
cmpl $RW_READER, 8(%esp) |
cmpl $RW_READER, 8(%esp) |
jne 2f |
jne 2f |
|
|
Line 139 STUB(rw_enter, rw_vector_enter) /* 0x0 |
|
Line 113 STUB(rw_enter, rw_vector_enter) /* 0x0 |
|
testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al |
testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al |
leal RW_READ_INCR(%eax), %ecx |
leal RW_READ_INCR(%eax), %ecx |
jnz,pn _C_LABEL(rw_vector_enter) |
jnz,pn _C_LABEL(rw_vector_enter) |
LOCK |
LOCK(lockpatch2) |
cmpxchgl %ecx, RW_OWNER(%edx) |
cmpxchgl %ecx, RW_OWNER(%edx) |
jnz,pn 1b |
jnz,pn 1b |
ret |
ret |
Line 150 STUB(rw_enter, rw_vector_enter) /* 0x0 |
|
Line 124 STUB(rw_enter, rw_vector_enter) /* 0x0 |
|
2: movl CPUVAR(CURLWP), %ecx |
2: movl CPUVAR(CURLWP), %ecx |
xorl %eax, %eax |
xorl %eax, %eax |
orl $RW_WRITE_LOCKED, %ecx |
orl $RW_WRITE_LOCKED, %ecx |
LOCK |
LOCK(lockpatch3) |
cmpxchgl %ecx, RW_OWNER(%edx) |
cmpxchgl %ecx, RW_OWNER(%edx) |
jnz,pn _C_LABEL(rw_vector_enter) |
jnz,pn _C_LABEL(rw_vector_enter) |
ret |
ret |
Line 160 STUB(rw_enter, rw_vector_enter) /* 0x0 |
|
Line 134 STUB(rw_enter, rw_vector_enter) /* 0x0 |
|
* |
* |
* Release one hold on a RW lock. |
* Release one hold on a RW lock. |
*/ |
*/ |
ALIGN64 |
ENTRY(rw_exit) |
|
movl 4(%esp), %edx |
STUB(rw_exit, rw_vector_exit) /* 0x0080, 61 bytes */ |
|
movl RW_OWNER(%edx), %eax |
movl RW_OWNER(%edx), %eax |
testb $RW_WRITE_LOCKED, %al |
testb $RW_WRITE_LOCKED, %al |
jnz 2f |
jnz 2f |
Line 175 STUB(rw_exit, rw_vector_exit) /* 0x008 |
|
Line 148 STUB(rw_exit, rw_vector_exit) /* 0x008 |
|
cmpl $RW_READ_INCR, %eax |
cmpl $RW_READ_INCR, %eax |
leal -RW_READ_INCR(%eax), %ecx |
leal -RW_READ_INCR(%eax), %ecx |
jb,pn 3f |
jb,pn 3f |
LOCK |
LOCK(lockpatch4) |
cmpxchgl %ecx, RW_OWNER(%edx) |
cmpxchgl %ecx, RW_OWNER(%edx) |
jnz,pn 1b |
jnz,pn 1b |
ret |
ret |
Line 186 STUB(rw_exit, rw_vector_exit) /* 0x008 |
|
Line 159 STUB(rw_exit, rw_vector_exit) /* 0x008 |
|
2: leal -RW_WRITE_LOCKED(%eax), %ecx |
2: leal -RW_WRITE_LOCKED(%eax), %ecx |
subl CPUVAR(CURLWP), %ecx |
subl CPUVAR(CURLWP), %ecx |
jnz,pn 3f |
jnz,pn 3f |
LOCK |
LOCK(lockpatch5) |
cmpxchgl %ecx, RW_OWNER(%edx) |
cmpxchgl %ecx, RW_OWNER(%edx) |
jnz,pn 3f |
jnz,pn 3f |
ret |
ret |
Line 203 STUB(rw_exit, rw_vector_exit) /* 0x008 |
|
Line 176 STUB(rw_exit, rw_vector_exit) /* 0x008 |
|
* |
* |
* Acquire a spin mutex and post a load fence. |
* Acquire a spin mutex and post a load fence. |
*/ |
*/ |
ALIGN64 |
ENTRY(mutex_spin_enter) |
|
movl 4(%esp), %edx |
STUB(mutex_spin_enter, mutex_vector_enter) /* 0x00c0, 51 bytes */ |
movl CPUVAR(ILEVEL), %ecx |
movl CPUVAR(SELF150), %eax |
subl $1, CPUVAR(MTX_COUNT) /* decl does not set CF */ |
movl (CPU_INFO_ILEVEL-0x150)(%eax), %ecx |
|
subl $1, (CPU_INFO_MTX_COUNT-0x150)(%eax)/* decl does not set CF */ |
|
jnc 1f |
jnc 1f |
movl %ecx, (CPU_INFO_MTX_OLDSPL-0x150)(%eax) |
movl %ecx, CPUVAR(MTX_OLDSPL) |
1: movb MTX_IPL(%edx), %ch |
1: movb MTX_IPL(%edx), %ch |
cmpb %ch, %cl |
cmpb %ch, %cl |
jg,pn 2f |
jg,pn 2f |
movb %ch, (CPU_INFO_ILEVEL-0x150)(%eax)/* splraiseipl() */ |
movb %ch, CPUVAR(ILEVEL) /* splraiseipl() */ |
2: |
2: |
#if defined(FULL) |
#ifdef FULL |
mov $0x0100, %eax /* new + expected value */ |
mov $0x0100, %eax /* new + expected value */ |
LOCK |
LOCK(lockpatch11) |
cmpxchgb %ah, MTX_LOCK(%edx) /* lock it */ |
cmpxchgb %ah, MTX_LOCK(%edx) /* lock it */ |
jnz,pn _C_LABEL(mutex_spin_retry) |
jnz,pn _C_LABEL(mutex_spin_retry) |
#endif |
#endif |
Line 228 STUB(mutex_spin_enter, mutex_vector_ente |
|
Line 199 STUB(mutex_spin_enter, mutex_vector_ente |
|
LABEL(mutex_spin_enter_end) |
LABEL(mutex_spin_enter_end) |
|
|
/* |
/* |
* void mutex_spin_exit(kmutex_t *mtx); |
|
* |
|
* Release a spin mutex and post a store fence. |
* Release a spin mutex and post a store fence. |
*/ |
*/ |
ALIGN64 |
ENTRY(mutex_spin_exit) |
|
movl 4(%esp), %edx |
STUB(mutex_spin_exit, mutex_vector_exit) /* 0x0100, 50 bytes */ |
movl CPUVAR(MTX_OLDSPL), %ecx |
#if defined(DIAGNOSTIC) |
incl CPUVAR(MTX_COUNT) |
movl $0x0001, %eax /* new + expected value */ |
movb $0, MTX_LOCK(%edx) /* zero */ |
cmpxchgb %ah, MTX_LOCK(%edx) |
|
jnz,pn _C_LABEL(mutex_vector_exit) |
|
#elif defined(MULTIPROCESSOR) |
|
movb $0x00,MTX_LOCK(%edx) |
|
#endif |
|
movl CPUVAR(SELF150), %eax |
|
movl (CPU_INFO_MTX_OLDSPL-0x150)(%eax), %ecx |
|
incl (CPU_INFO_MTX_COUNT-0x150)(%eax) |
|
jnz 1f |
jnz 1f |
cmpl (CPU_INFO_ILEVEL-0x150)(%eax), %ecx |
movl %fs:CPU_INFO_IUNMASK(,%ecx,4), %edx |
movl %ecx, 4(%esp) |
|
jae 1f |
|
movl (CPU_INFO_IUNMASK-0x150)(%eax,%ecx,4), %edx |
|
cli |
cli |
testl (CPU_INFO_IPENDING-0x150)(%eax), %edx |
testl CPUVAR(IPENDING), %edx |
|
movl %ecx, 4(%esp) |
jnz _C_LABEL(Xspllower) /* does sti */ |
jnz _C_LABEL(Xspllower) /* does sti */ |
movl %ecx, (CPU_INFO_ILEVEL-0x150)(%eax) |
movl %ecx, CPUVAR(ILEVEL) |
sti |
sti |
1: ret |
1: ret |
|
nop /* XXX round up */ |
ALIGN64 |
.align 32 |
LABEL(mutex_spin_exit_end) |
LABEL(mutex_spin_exit_end) |
|
|
#if !defined(I386_CPU) && defined(I686_CPU) && !defined(DIAGNOSTIC) |
|
|
|
/* |
/* |
* Patch for i686 CPUs where cli/sti is prohibitavely expensive. |
* Patch for i686 CPUs where cli/sti is prohibitavely expensive. |
* Must be the same size as mutex_spin_exit(). |
* Must be the same size as mutex_spin_exit(). |
*/ |
*/ |
ALIGN64 |
ENTRY(i686_mutex_spin_exit) |
|
|
ENTRY(i686_mutex_spin_exit) /* 64 bytes */ |
|
mov 4(%esp),%edx |
mov 4(%esp),%edx |
xorl %eax,%eax |
movl CPUVAR(MTX_OLDSPL), %ecx |
pushl %edi |
incl CPUVAR(MTX_COUNT) |
fs |
movb %ch, MTX_LOCK(%edx) /* zero */ |
movl (CPU_INFO_SELF150)(%eax), %edi /* now splx() */ |
|
pushl %ebx |
|
movl (CPU_INFO_MTX_OLDSPL-0x150)(%edi), %ecx |
|
incl (CPU_INFO_MTX_COUNT-0x150)(%edi) |
|
movb %al, MTX_LOCK(%edx) /* zero */ |
|
movl (CPU_INFO_ILEVEL-0x150)(%edi), %edx |
|
jnz 1f |
jnz 1f |
cmpl %edx, %ecx /* new level is lower? */ |
pushl %ebx |
movl (CPU_INFO_IPENDING-0x150)(%edi), %eax |
0: |
jae,pn 1f |
movl CPUVAR(IPENDING), %eax |
testl %eax,(CPU_INFO_IUNMASK-0x150)(%edi,%ecx,4) |
testl %eax, %fs:CPU_INFO_IUNMASK(,%ecx,4) |
movl %eax, %ebx |
movl %eax, %ebx |
/* |
/* |
* On a P4 this jump is cheaper than patching in junk using |
* On a P4 this jump is cheaper than patching in junk using |
* cmovnz. Is cmpxchg expensive if it fails? |
* cmovnz. Is cmpxchg expensive if it fails? |
*/ |
*/ |
jnz,pn 2f |
jnz 2f |
cmpxchg8b (CPU_INFO_ISTATE-0x150)(%edi) /* swap in new ilevel */ |
cmpxchg8b CPUVAR(ISTATE) /* swap in new ilevel */ |
jnz,pn 2f |
jnz 0b |
1: |
|
popl %ebx |
popl %ebx |
popl %edi |
1: |
ret |
ret |
2: |
2: |
popl %ebx |
popl %ebx |
popl %edi |
|
movl %ecx,4(%esp) |
movl %ecx,4(%esp) |
LABEL(i686_mutex_spin_exit_patch) |
LABEL(i686_mutex_spin_exit_patch) |
jmp _C_LABEL(Xspllower) |
jmp _C_LABEL(Xspllower) |
ALIGN64 |
.align 32 |
LABEL(i686_mutex_spin_exit_end) |
LABEL(i686_mutex_spin_exit_end) |
|
|
#endif /* !defined(I386_CPU) && defined(I686_CPU) && !defined(DIAGNOSTIC) */ |
|
|
|
#else /* !__XEN__ */ |
#else /* !__XEN__ */ |
|
|
/* For now; strong alias not working for some reason. */ |
/* For now; strong alias not working for some reason. */ |
NENTRY(mutex_spin_enter) |
ENTRY(mutex_spin_enter) |
jmp _C_LABEL(mutex_vector_enter) |
jmp _C_LABEL(mutex_vector_enter) |
|
|
NENTRY(mutex_spin_exit) |
ENTRY(mutex_spin_exit) |
jmp _C_LABEL(mutex_vector_exit) |
jmp _C_LABEL(mutex_vector_exit) |
|
|
#endif /* !__XEN__ */ |
#endif /* !__XEN__ */ |
Line 325 NENTRY(mutex_spin_exit) |
|
Line 270 NENTRY(mutex_spin_exit) |
|
* |
* |
* Perform an atomic compare-and-set operation. |
* Perform an atomic compare-and-set operation. |
*/ |
*/ |
ALIGN64 |
ENTRY(_lock_cas) |
|
mov 4(%esp),%edx |
STUB(_lock_cas, _80386_lock_cas) /* 32 bytes */ |
|
movl 8(%esp), %eax |
movl 8(%esp), %eax |
movl 12(%esp), %ecx |
movl 12(%esp), %ecx |
LOCK |
LOCK(lockpatch6) |
cmpxchgl %ecx, (%edx) |
cmpxchgl %ecx, (%edx) |
movl $0, %eax |
movl $0, %eax |
setz %al |
setz %al |
ret |
ret |
|
|
#ifdef I386_CPU |
|
/* |
|
* Since we can't do compare-and-exchange atomically with an 80386, we must |
|
* disable interrupts in order to support preemption. On the i386 this is |
|
* cheap to do. For other architectures a restartable sequence is usually |
|
* a better option. |
|
*/ |
|
_80386_lock_cas: |
|
movl 8(%esp), %eax |
|
movl 12(%esp), %ecx |
|
cli |
|
cmpl %eax, (%edx) |
|
jne 1f |
|
movl %ecx, (%edx) |
|
movb $1, %al |
|
sti |
|
ret |
|
|
|
1: sti |
|
xorl %eax, %eax |
|
ret |
|
#endif /* I386_CPU */ |
|
|
|
/* |
/* |
* Memory barrier operations, may be patched at runtime. |
* Memory barrier operations, may be patched at runtime. |
*/ |
*/ |
.align 8 |
.align 8 |
|
|
NENTRY(mb_read) |
ENTRY(mb_read) |
lock |
LOCK(lockpatch7) |
addl $0, 0(%esp) |
addl $0, 0(%esp) |
ret |
ret |
END(mb_read_end, 8) |
END(mb_read_end, 8) |
|
|
NENTRY(mb_write) |
ENTRY(mb_write) |
/* Nothing at the moment. */ |
nop |
ret |
ret |
END(mb_write_end, 8) |
END(mb_write_end, 8) |
|
|
NENTRY(mb_memory) |
ENTRY(mb_memory) |
lock |
LOCK(lockpatch8) |
addl $0, 0(%esp) |
addl $0, 0(%esp) |
ret |
ret |
END(mb_memory_end, 8) |
END(mb_memory_end, 8) |
|
|
#ifdef I686_CPU |
ENTRY(sse2_mb_read) |
NENTRY(sse2_mb_read) |
|
lfence |
lfence |
ret |
ret |
END(sse2_mb_read_end, 8) |
END(sse2_mb_read_end, 8) |
|
|
NENTRY(sse2_mb_memory) |
ENTRY(sse2_mb_memory) |
mfence |
mfence |
ret |
ret |
END(sse2_mb_memory_end, 8) |
END(sse2_mb_memory_end, 8) |
#endif /* I686_CPU */ |
|
|
|
/* |
/* |
* Make sure code after the ret is properly encoded with nopness |
* Patchpoints to replace with NOP when ncpu == 1. |
* by gas, or could stall newer processors. |
|
*/ |
*/ |
|
#ifndef LOCKDEBUG |
NENTRY(x86_mb_nop) |
LABEL(x86_lockpatch) |
ret |
.long lockpatch1, lockpatch2, lockpatch3, lockpatch4 |
END(x86_mb_nop_end, 8) |
.long lockpatch5, lockpatch6, lockpatch7, lockpatch8 |
|
#if defined(FULL) && !defined(__XEN__) |
|
.long lockpatch11 |
|
#endif |
|
.long 0 |
|
#endif |