version 1.9, 2008/02/06 22:27:09 |
version 1.9.10.4, 2010/08/11 22:51:32 |
|
|
|
|
#define GET_CURPCB(reg) \ |
#define GET_CURPCB(reg) \ |
movq CPUVAR(CURLWP), reg; \ |
movq CPUVAR(CURLWP), reg; \ |
movq L_ADDR(reg), reg |
movq L_PCB(reg), reg |
|
|
/* |
/* |
* These are arranged so that the abnormal case is a forwards |
* These are arranged so that the abnormal case is a forwards |
|
|
call _C_LABEL(do_pmap_load) ; \ |
call _C_LABEL(do_pmap_load) ; \ |
jmp 98b |
jmp 98b |
|
|
|
/* |
|
* The following primitives are to copy regions of memory. |
|
* Label must be before all copy functions. |
|
*/ |
|
.text |
|
|
|
x86_copyfunc_start: .globl x86_copyfunc_start |
|
|
|
/* |
|
* Handle deferred pmap switch. We must re-enable preemption without |
|
* making a function call, so that the program counter is visible to |
|
* cpu_kpreempt_exit(). It can then know if it needs to restore the |
|
* pmap on returning, because a preemption occurred within one of the |
|
* copy functions. |
|
*/ |
ENTRY(do_pmap_load) |
ENTRY(do_pmap_load) |
#if 1 |
|
pushq %rbp |
pushq %rbp |
movq %rsp,%rbp |
movq %rsp,%rbp |
#endif |
|
pushq %rdi |
pushq %rdi |
pushq %rsi |
pushq %rsi |
pushq %rdx |
pushq %rdx |
pushq %rcx |
pushq %rcx |
|
pushq %rbx |
|
movq CPUVAR(CURLWP), %rbx |
|
1: |
|
incl L_NOPREEMPT(%rbx) |
call _C_LABEL(pmap_load) |
call _C_LABEL(pmap_load) |
|
decl L_NOPREEMPT(%rbx) |
|
jnz 2f |
|
cmpl $0, L_DOPREEMPT(%rbx) |
|
jz 2f |
|
xorq %rdi, %rdi |
|
call _C_LABEL(kpreempt) |
|
2: |
|
cmpl $0, CPUVAR(WANT_PMAPLOAD) |
|
jnz 1b |
|
popq %rbx |
popq %rcx |
popq %rcx |
popq %rdx |
popq %rdx |
popq %rsi |
popq %rsi |
popq %rdi |
popq %rdi |
#if 1 |
|
leaveq |
leaveq |
#endif |
|
ret |
ret |
|
|
/* |
/* |
|
* int kcopy(const void *from, void *to, size_t len); |
|
* Copy len bytes, abort on fault. |
|
* |
* Copy routines from and to userland, plus a few more. See the |
* Copy routines from and to userland, plus a few more. See the |
* section 9 manpages for info. Some cases can be optimized more. |
* section 9 manpages for info. Some cases can be optimized more. |
* |
* |
Line 97 ENTRY(do_pmap_load) |
|
Line 125 ENTRY(do_pmap_load) |
|
* be ably to do cache-line size copies.... |
* be ably to do cache-line size copies.... |
*/ |
*/ |
|
|
/* |
|
* XXXfvdl appears only to be used by pccons. |
|
* |
|
* fillw(short pattern, void *addr, size_t len); |
|
* Write len copies of pattern at addr. |
|
* appears to be used by pccons. |
|
*/ |
|
ENTRY(fillw) |
|
movl %edi,%eax |
|
movq %rsi,%rdi |
|
movw %ax,%cx |
|
rorl $16,%eax |
|
movw %cx,%ax |
|
movq %rdx,%rcx |
|
shrq %rcx |
|
rep |
|
stosl |
|
movq %rdx,%rcx |
|
andq $1,%rcx |
|
rep |
|
stosw |
|
ret |
|
|
|
ENTRY(kcopy) |
ENTRY(kcopy) |
GET_CURPCB(%rax) |
|
pushq PCB_ONFAULT(%rax) |
|
leaq _C_LABEL(kcopy_fault)(%rip),%r11 |
|
movq %r11, PCB_ONFAULT(%rax) |
|
|
|
xchgq %rdi,%rsi |
xchgq %rdi,%rsi |
movq %rdx,%rcx |
movq %rdx,%rcx |
|
.Lkcopy_start: |
movq %rdi,%rax |
movq %rdi,%rax |
subq %rsi,%rax |
subq %rsi,%rax |
cmpq %rcx,%rax # overlapping? |
cmpq %rcx,%rax # overlapping? |
|
|
rep |
rep |
movsb |
movsb |
|
|
GET_CURPCB(%rdx) |
|
popq PCB_ONFAULT(%rdx) |
|
xorq %rax,%rax |
xorq %rax,%rax |
ret |
ret |
|
|
|
# Using 'rep movs' to copy backwards is not as fast as for forwards copies |
|
# and ought not be done when the copy doesn't acually overlap. |
|
# However kcopy() isn't used any that looks even vaguely used often. |
|
# I'm also not sure it is ever asked to do overlapping copies! |
|
|
1: addq %rcx,%rdi # copy backward |
1: addq %rcx,%rdi # copy backward |
addq %rcx,%rsi |
addq %rcx,%rsi |
std |
std |
|
|
rep |
rep |
movsq |
movsq |
cld |
cld |
|
.Lkcopy_end: |
GET_CURPCB(%rdx) |
|
popq PCB_ONFAULT(%rdx) |
|
xorq %rax,%rax |
xorq %rax,%rax |
ret |
ret |
|
|
ENTRY(copyout) |
ENTRY(copyout) |
DEFERRED_SWITCH_CHECK |
DEFERRED_SWITCH_CHECK |
pushq $0 |
|
|
|
xchgq %rdi,%rsi |
xchgq %rdi,%rsi # kernel address to %rsi, user to %rdi |
movq %rdx,%rax |
movq %rdx,%rax # save transfer length (bytes) |
|
|
movq %rdi,%rdx |
addq %rdi,%rdx # end address to %rdx |
addq %rax,%rdx |
jc _C_LABEL(copy_efault) # jump if wraps |
jc _C_LABEL(copy_efault) |
|
movq $VM_MAXUSER_ADDRESS,%r8 |
movq $VM_MAXUSER_ADDRESS,%r8 |
cmpq %r8,%rdx |
cmpq %r8,%rdx |
ja _C_LABEL(copy_efault) |
ja _C_LABEL(copy_efault) # jump if end in kernel space |
|
|
GET_CURPCB(%rdx) |
.Lcopyout_start: |
leaq _C_LABEL(copy_fault)(%rip),%r11 |
movq %rax,%rcx # length |
movq %r11,PCB_ONFAULT(%rdx) |
shrq $3,%rcx # count of 8-byte words |
|
|
movq %rax,%rcx |
|
shrq $3,%rcx |
|
rep |
rep |
movsq |
movsq # copy from %rsi to %rdi |
movb %al,%cl |
movb %al,%cl |
andb $7,%cl |
andb $7,%cl # remaining number of bytes |
rep |
rep |
movsb |
movsb # copy remaining bytes |
|
.Lcopyout_end: |
popq PCB_ONFAULT(%rdx) |
|
xorl %eax,%eax |
xorl %eax,%eax |
ret |
ret |
DEFERRED_SWITCH_CALL |
DEFERRED_SWITCH_CALL |
|
|
ENTRY(copyin) |
ENTRY(copyin) |
DEFERRED_SWITCH_CHECK |
DEFERRED_SWITCH_CHECK |
GET_CURPCB(%rax) |
|
pushq $0 |
|
leaq _C_LABEL(copy_fault)(%rip),%r11 |
|
movq %r11,PCB_ONFAULT(%rax) |
|
|
|
xchgq %rdi,%rsi |
xchgq %rdi,%rsi |
movq %rdx,%rax |
movq %rdx,%rax |
|
|
movq %rsi,%rdx |
addq %rsi,%rdx # Check source address not wrapped |
addq %rax,%rdx |
|
jc _C_LABEL(copy_efault) |
jc _C_LABEL(copy_efault) |
movq $VM_MAXUSER_ADDRESS,%r8 |
movq $VM_MAXUSER_ADDRESS,%r8 |
cmpq %r8,%rdx |
cmpq %r8,%rdx |
ja _C_LABEL(copy_efault) |
ja _C_LABEL(copy_efault) # j if end in kernel space |
|
|
|
.Lcopyin_start: |
3: /* bcopy(%rsi, %rdi, %rax); */ |
3: /* bcopy(%rsi, %rdi, %rax); */ |
movq %rax,%rcx |
movq %rax,%rcx |
shrq $3,%rcx |
shrq $3,%rcx |
|
|
andb $7,%cl |
andb $7,%cl |
rep |
rep |
movsb |
movsb |
|
.Lcopyin_end: |
GET_CURPCB(%rdx) |
|
popq PCB_ONFAULT(%rdx) |
|
xorl %eax,%eax |
xorl %eax,%eax |
ret |
ret |
DEFERRED_SWITCH_CALL |
DEFERRED_SWITCH_CALL |
Line 244 NENTRY(copy_efault) |
|
Line 233 NENTRY(copy_efault) |
|
*/ |
*/ |
|
|
NENTRY(kcopy_fault) |
NENTRY(kcopy_fault) |
GET_CURPCB(%rdx) |
|
popq PCB_ONFAULT(%rdx) |
|
ret |
ret |
|
|
NENTRY(copy_fault) |
NENTRY(copy_fault) |
GET_CURPCB(%rdx) |
|
popq PCB_ONFAULT(%rdx) |
|
ret |
ret |
|
|
ENTRY(copyoutstr) |
ENTRY(copyoutstr) |
DO_DEFERRED_SWITCH |
DEFERRED_SWITCH_CHECK |
xchgq %rdi,%rsi |
xchgq %rdi,%rsi |
movq %rdx,%r8 |
movq %rdx,%r8 |
movq %rcx,%r9 |
movq %rcx,%r9 |
|
|
5: GET_CURPCB(%rax) |
|
leaq _C_LABEL(copystr_fault)(%rip),%r11 |
|
movq %r11,PCB_ONFAULT(%rax) |
|
/* |
/* |
* Get min(%rdx, VM_MAXUSER_ADDRESS-%rdi). |
* Get min(%rdx, VM_MAXUSER_ADDRESS-%rdi). |
*/ |
*/ |
Line 272 ENTRY(copyoutstr) |
|
Line 254 ENTRY(copyoutstr) |
|
jae 1f |
jae 1f |
movq %rax,%rdx |
movq %rax,%rdx |
movq %rax,%r8 |
movq %rax,%r8 |
|
.Lcopyoutstr_start: |
1: incq %rdx |
1: incq %rdx |
|
|
1: decq %rdx |
1: decq %rdx |
Line 281 ENTRY(copyoutstr) |
|
Line 263 ENTRY(copyoutstr) |
|
stosb |
stosb |
testb %al,%al |
testb %al,%al |
jnz 1b |
jnz 1b |
|
.Lcopyoutstr_end: |
/* Success -- 0 byte reached. */ |
/* Success -- 0 byte reached. */ |
decq %rdx |
decq %rdx |
xorq %rax,%rax |
xorq %rax,%rax |
Line 293 ENTRY(copyoutstr) |
|
Line 275 ENTRY(copyoutstr) |
|
jae _C_LABEL(copystr_efault) |
jae _C_LABEL(copystr_efault) |
movq $ENAMETOOLONG,%rax |
movq $ENAMETOOLONG,%rax |
jmp copystr_return |
jmp copystr_return |
|
DEFERRED_SWITCH_CALL |
|
|
ENTRY(copyinstr) |
ENTRY(copyinstr) |
DO_DEFERRED_SWITCH |
DEFERRED_SWITCH_CHECK |
xchgq %rdi,%rsi |
xchgq %rdi,%rsi |
movq %rdx,%r8 |
movq %rdx,%r8 |
movq %rcx,%r9 |
movq %rcx,%r9 |
|
|
GET_CURPCB(%rcx) |
|
leaq _C_LABEL(copystr_fault)(%rip),%r11 |
|
movq %r11,PCB_ONFAULT(%rcx) |
|
|
|
/* |
/* |
* Get min(%rdx, VM_MAXUSER_ADDRESS-%rsi). |
* Get min(%rdx, VM_MAXUSER_ADDRESS-%rsi). |
*/ |
*/ |
Line 314 ENTRY(copyinstr) |
|
Line 293 ENTRY(copyinstr) |
|
jae 1f |
jae 1f |
movq %rax,%rdx |
movq %rax,%rdx |
movq %rax,%r8 |
movq %rax,%r8 |
|
.Lcopyinstr_start: |
1: incq %rdx |
1: incq %rdx |
|
|
1: decq %rdx |
1: decq %rdx |
Line 323 ENTRY(copyinstr) |
|
Line 302 ENTRY(copyinstr) |
|
stosb |
stosb |
testb %al,%al |
testb %al,%al |
jnz 1b |
jnz 1b |
|
.Lcopyinstr_end: |
|
|
/* Success -- 0 byte reached. */ |
/* Success -- 0 byte reached. */ |
decq %rdx |
decq %rdx |
Line 335 ENTRY(copyinstr) |
|
Line 315 ENTRY(copyinstr) |
|
jae _C_LABEL(copystr_efault) |
jae _C_LABEL(copystr_efault) |
movq $ENAMETOOLONG,%rax |
movq $ENAMETOOLONG,%rax |
jmp copystr_return |
jmp copystr_return |
|
DEFERRED_SWITCH_CALL |
|
|
ENTRY(copystr_efault) |
ENTRY(copystr_efault) |
movl $EFAULT,%eax |
movl $EFAULT,%eax |
Line 342 ENTRY(copystr_efault) |
|
Line 323 ENTRY(copystr_efault) |
|
ENTRY(copystr_fault) |
ENTRY(copystr_fault) |
copystr_return: |
copystr_return: |
/* Set *lencopied and return %eax. */ |
/* Set *lencopied and return %eax. */ |
GET_CURPCB(%rcx) |
|
movq $0,PCB_ONFAULT(%rcx) |
|
testq %r9,%r9 |
testq %r9,%r9 |
jz 8f |
jz 8f |
subq %rdx,%r8 |
subq %rdx,%r8 |
|
|
DEFERRED_SWITCH_CALL |
DEFERRED_SWITCH_CALL |
|
|
ENTRY(fuswintr) |
ENTRY(fuswintr) |
cmpq $TLBSTATE_VALID, CPUVAR(TLBSTATE) |
cmpl $TLBSTATE_VALID, CPUVAR(TLBSTATE) |
jnz _C_LABEL(fusuaddrfault) |
jnz _C_LABEL(fusuaddrfault) |
movq $VM_MAXUSER_ADDRESS-2,%r11 |
movq $VM_MAXUSER_ADDRESS-2,%r11 |
cmpq %r11,%rdi |
cmpq %r11,%rdi |
|
|
cmpq %r11,%rdi |
cmpq %r11,%rdi |
ja _C_LABEL(fusuaddrfault) |
ja _C_LABEL(fusuaddrfault) |
GET_CURPCB(%rcx) |
GET_CURPCB(%rcx) |
leaq _C_LABEL(fusuintrfailure)(%rip),%r11 |
leaq _C_LABEL(fusufailure)(%rip),%r11 |
movq %r11,PCB_ONFAULT(%rcx) |
movq %r11,PCB_ONFAULT(%rcx) |
movzbl (%rdi),%eax |
movzbl (%rdi),%eax |
movq $0,PCB_ONFAULT(%rcx) |
movq $0,PCB_ONFAULT(%rcx) |
|
|
DEFERRED_SWITCH_CALL |
DEFERRED_SWITCH_CALL |
|
|
ENTRY(suswintr) |
ENTRY(suswintr) |
cmpq $TLBSTATE_VALID, CPUVAR(TLBSTATE) |
cmpl $TLBSTATE_VALID, CPUVAR(TLBSTATE) |
jnz _C_LABEL(fusuaddrfault) |
jnz _C_LABEL(fusuaddrfault) |
movq $VM_MAXUSER_ADDRESS-2,%r11 |
movq $VM_MAXUSER_ADDRESS-2,%r11 |
cmpq %r11,%rdi |
cmpq %r11,%rdi |
|
|
GET_CURPCB(%rcx) |
GET_CURPCB(%rcx) |
leaq _C_LABEL(fusuintrfailure)(%rip),%r11 |
leaq _C_LABEL(fusuintrfailure)(%rip),%r11 |
movq %r11,PCB_ONFAULT(%rcx) |
movq %r11,PCB_ONFAULT(%rcx) |
|
|
movw %si,(%rdi) |
movw %si,(%rdi) |
xorq %rax,%rax |
xorq %rax,%rax |
movq %rax,PCB_ONFAULT(%rcx) |
movq %rax,PCB_ONFAULT(%rcx) |
Line 512 ENTRY(fusufailure) |
|
Line 490 ENTRY(fusufailure) |
|
ENTRY(fusuaddrfault) |
ENTRY(fusuaddrfault) |
movl $-1,%eax |
movl $-1,%eax |
ret |
ret |
|
|
|
/* |
|
* Compare-and-swap the 64-bit integer in the user-space. |
|
* |
|
* int ucas_64(volatile int64_t *uptr, int64_t old, int64_t new, int64_t *ret); |
|
*/ |
|
ENTRY(ucas_64) |
|
DEFERRED_SWITCH_CHECK |
|
/* Fail if kernel-space */ |
|
movq $VM_MAXUSER_ADDRESS-8, %r8 |
|
cmpq %r8, %rdi |
|
ja _C_LABEL(ucas_fault) |
|
movq %rsi, %rax |
|
.Lucas64_start: |
|
/* Perform the CAS */ |
|
lock |
|
cmpxchgq %rdx, (%rdi) |
|
.Lucas64_end: |
|
/* |
|
* Note: %rax is "old" value. |
|
* Set the return values. |
|
*/ |
|
movq %rax, (%rcx) |
|
xorq %rax, %rax |
|
ret |
|
DEFERRED_SWITCH_CALL |
|
|
|
/* |
|
* int ucas_32(volatile int32_t *uptr, int32_t old, int32_t new, int32_t *ret); |
|
*/ |
|
ENTRY(ucas_32) |
|
DEFERRED_SWITCH_CHECK |
|
/* Fail if kernel-space */ |
|
movq $VM_MAXUSER_ADDRESS-4, %r8 |
|
cmpq %r8, %rdi |
|
ja _C_LABEL(ucas_fault) |
|
movl %esi, %eax |
|
.Lucas32_start: |
|
/* Perform the CAS */ |
|
lock |
|
cmpxchgl %edx, (%rdi) |
|
.Lucas32_end: |
|
/* |
|
* Note: %eax is "old" value. |
|
* Set the return values. |
|
*/ |
|
movl %eax, (%rcx) |
|
xorq %rax, %rax |
|
ret |
|
DEFERRED_SWITCH_CALL |
|
|
|
/* |
|
* Fault handler for ucas_*(). |
|
* Just return the error set by trap(). |
|
*/ |
|
NENTRY(ucas_fault) |
|
ret |
|
|
|
/* |
|
* int ucas_ptr(volatile void **uptr, void *old, void *new, void **ret); |
|
* int ucas_int(volatile int *uptr, int old, intnew, int *ret); |
|
*/ |
|
STRONG_ALIAS(ucas_ptr, ucas_64) |
|
STRONG_ALIAS(ucas_int, ucas_32) |
|
|
|
/* |
|
* Label must be after all copy functions. |
|
*/ |
|
x86_copyfunc_end: .globl x86_copyfunc_end |
|
|
|
/* |
|
* Fault table of copy functions for trap(). |
|
*/ |
|
.section ".rodata" |
|
.globl _C_LABEL(onfault_table) |
|
_C_LABEL(onfault_table): |
|
.quad .Lcopyin_start |
|
.quad .Lcopyin_end |
|
.quad _C_LABEL(copy_fault) |
|
|
|
.quad .Lcopyout_start |
|
.quad .Lcopyout_end |
|
.quad _C_LABEL(copy_fault) |
|
|
|
.quad .Lkcopy_start |
|
.quad .Lkcopy_end |
|
.quad _C_LABEL(kcopy_fault) |
|
|
|
.quad .Lcopyoutstr_start |
|
.quad .Lcopyoutstr_end |
|
.quad _C_LABEL(copystr_fault) |
|
|
|
.quad .Lcopyinstr_start |
|
.quad .Lcopyinstr_end |
|
.quad _C_LABEL(copystr_fault) |
|
|
|
.quad .Lucas64_start |
|
.quad .Lucas64_end |
|
.quad _C_LABEL(ucas_fault) |
|
|
|
.quad .Lucas32_start |
|
.quad .Lucas32_end |
|
.quad _C_LABEL(ucas_fault) |
|
|
|
.quad 0 /* terminate */ |
|
|
|
.text |