/* $NetBSD: spl.S,v 1.6.14.2 2007/10/03 19:22:10 garbled Exp $ */
/*
* Copyright (c) 2003 Wasabi Systems, Inc.
* All rights reserved.
*
* Written by Frank van der Linden for Wasabi Systems, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed for the NetBSD Project by
* Wasabi Systems, Inc.
* 4. The name of Wasabi Systems, Inc. may not be used to endorse
* or promote products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Copyright (c) 1998, 2007 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Charles M. Hannum and Andrew Doran.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by the NetBSD
* Foundation, Inc. and its contributors.
* 4. Neither the name of The NetBSD Foundation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "opt_ddb.h"
#define ALIGN_TEXT .align 16,0x90
#include <machine/asm.h>
#include <machine/psl.h>
#include <machine/trap.h>
#include <machine/segments.h>
#include <machine/frameasm.h>
#include "assym.h"
.text
/*
* int splraise(int s);
*/
.align 32
ENTRY(splraise)
movl CPUVAR(ILEVEL),%eax
cmpl %edi,%eax
cmoval %eax,%edi
movl %edi,CPUVAR(ILEVEL)
ret
/*
* void softintr(int sir);
*
* Software interrupt registration.
*/
.align 32
ENTRY(softintr)
movl %edi,%ecx
movl $1,%eax
shll %cl,%eax
orl %eax,CPUVAR(IPENDING)
ret
/*
* void spllower(int s);
*
* Must be the same size as i686_spllower(). This must use
* pushf/cli/popf as it is used early in boot where interrupts
* are disabled via eflags/IE.
*/
.align 64
ENTRY(spllower)
cmpl CPUVAR(ILEVEL), %edi
jae 1f
movl CPUVAR(IUNMASK)(,%edi,4), %edx
pushf
cli
testl CPUVAR(IPENDING), %edx
jnz 2f
movl %edi, CPUVAR(ILEVEL)
popf
1:
rep
ret
2:
popf
jmp _C_LABEL(Xspllower)
.align 64
LABEL(spllower_end)
/*
* void amd64_spllower(int s);
*
* For cmpxchg8b, edx/ecx are the high words and eax/ebx the low.
*
* edx : eax = old level / old ipending
* ecx : ebx = new level / old ipending
*/
.align 64
ENTRY(amd64_spllower)
movl CPUVAR(ILEVEL),%edx
cmpl %edx,%edi /* new level is lower? */
pushq %rbx
jae,pn 1f
0:
movl CPUVAR(IPENDING),%eax
movl %edi,%ecx
testl %eax,CPUVAR(IUNMASK)(,%rcx,4)/* deferred interrupts? */
movl %eax,%ebx
/*
* On the P4 this jump is cheaper than patching in junk
* using cmov. Is cmpxchg expensive if it fails?
*/
jnz,pn 2f
cmpxchg8b CPUVAR(ISTATE) /* swap in new ilevel */
jnz,pn 0b
1:
popq %rbx
ret
2:
popq %rbx
LABEL(amd64_spllower_patch)
jmp _C_LABEL(Xspllower)
.align 64
LABEL(amd64_spllower_end)
/*
* void Xspllower(int s);
*
* Process pending interrupts.
*
* Important registers:
* ebx - cpl
* r13 - address to resume loop at
*
* It is important that the bit scan instruction is bsr, it will get
* the highest 2 bits (currently the IPI and clock handlers) first,
* to avoid deadlocks where one CPU sends an IPI, another one is at
* splipi() and defers it, lands in here via splx(), and handles
* a lower-prio one first, which needs to take the kernel lock -->
* the sending CPU will never see the that CPU accept the IPI
* (see pmap_tlb_shootnow).
*/
nop
.align 4 /* Avoid confusion with amd64_spllower_end */
IDTVEC(spllower)
pushq %rbx
pushq %r13
pushq %r12
movl %edi,%ebx
leaq 1f(%rip),%r13 # address to resume loop at
1: movl %ebx,%eax # get cpl
movl CPUVAR(IUNMASK)(,%rax,4),%eax
cli
andl CPUVAR(IPENDING),%eax # any non-masked bits left?
jz 2f
bsrl %eax,%eax
btrl %eax,CPUVAR(IPENDING)
movq CPUVAR(ISOURCES)(,%rax,8),%rax
jmp *IS_RECURSE(%rax)
2:
movl %ebx,CPUVAR(ILEVEL)
sti
popq %r12
popq %r13
popq %rbx
ret
/*
* Handle return from interrupt after device handler finishes.
*
* Important registers:
* ebx - cpl to restore
* r13 - address to resume loop at
*/
IDTVEC(doreti)
popq %rbx # get previous priority
decl CPUVAR(IDEPTH)
leaq 1f(%rip),%r13
1: movl %ebx,%eax
movl CPUVAR(IUNMASK)(,%rax,4),%eax
cli
andl CPUVAR(IPENDING),%eax
jz 2f
bsrl %eax,%eax # slow, but not worth optimizing
btrl %eax,CPUVAR(IPENDING)
movq CPUVAR(ISOURCES)(,%rax, 8),%rax
jmp *IS_RESUME(%rax)
2: /* Check for ASTs on exit to user mode. */
movl %ebx,CPUVAR(ILEVEL)
movq CPUVAR(CURLWP),%r14
5: CHECK_ASTPENDING(%r14)
je 3f
testb $SEL_RPL,TF_CS(%rsp)
jz 3f
CLEAR_ASTPENDING(%r14)
sti
movl $T_ASTFLT,TF_TRAPNO(%rsp) /* XXX undo later.. */
/* Pushed T_ASTFLT into tf_trapno on entry. */
movq %rsp,%rdi
call _C_LABEL(trap)
cli
jmp 5b
3: INTRFASTEXIT