version 1.14.2.8, 2008/03/17 09:14:14 |
version 1.15, 2005/12/11 12:16:21 |
|
|
*/ |
*/ |
|
|
/* |
/* |
* Copyright (c) 2007 Manuel Bouyer. |
|
* |
|
* Redistribution and use in source and binary forms, with or without |
|
* modification, are permitted provided that the following conditions |
|
* are met: |
|
* 1. Redistributions of source code must retain the above copyright |
|
* notice, this list of conditions and the following disclaimer. |
|
* 2. Redistributions in binary form must reproduce the above copyright |
|
* notice, this list of conditions and the following disclaimer in the |
|
* documentation and/or other materials provided with the distribution. |
|
* 3. All advertising materials mentioning features or use of this software |
|
* must display the following acknowledgement: |
|
* This product includes software developed by Manuel Bouyer. |
|
* 4. The name of the author may not be used to endorse or promote products |
|
* derived from this software without specific prior written permission. |
|
* |
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
|
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
|
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
* |
|
*/ |
|
|
|
/* |
|
* Copyright (c) 2006 Mathieu Ropert <mro@adviseo.fr> |
|
* |
|
* Permission to use, copy, modify, and distribute this software for any |
|
* purpose with or without fee is hereby granted, provided that the above |
|
* copyright notice and this permission notice appear in all copies. |
|
* |
|
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
|
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
|
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR |
|
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
|
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
|
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF |
|
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
|
*/ |
|
|
|
/* |
|
* Copyright (c) 2001 Wasabi Systems, Inc. |
* Copyright (c) 2001 Wasabi Systems, Inc. |
* All rights reserved. |
* All rights reserved. |
* |
* |
|
|
|
|
|
|
/*- |
/*- |
* Copyright (c) 1998, 2000, 2007 The NetBSD Foundation, Inc. |
* Copyright (c) 1998, 2000 The NetBSD Foundation, Inc. |
* All rights reserved. |
* All rights reserved. |
* |
* |
* This code is derived from software contributed to The NetBSD Foundation |
* This code is derived from software contributed to The NetBSD Foundation |
|
|
* @(#)locore.s 7.3 (Berkeley) 5/13/91 |
* @(#)locore.s 7.3 (Berkeley) 5/13/91 |
*/ |
*/ |
|
|
/* |
|
* override user-land alignment before including asm.h |
|
*/ |
|
#define ALIGN_DATA .align 8 |
|
#define ALIGN_TEXT .align 16,0x90 |
|
#define _ALIGN_TEXT ALIGN_TEXT |
|
|
|
#include <machine/asm.h> |
|
|
|
#include "opt_ddb.h" |
#include "opt_ddb.h" |
#include "opt_ddbparam.h" |
#include "opt_ddbparam.h" |
|
#include "opt_user_ldt.h" |
|
#include "opt_multiprocessor.h" |
|
#include "opt_lockdebug.h" |
#include "opt_realmem.h" |
#include "opt_realmem.h" |
|
|
#include "opt_compat_netbsd.h" |
#include "opt_compat_netbsd.h" |
#include "opt_compat_netbsd32.h" |
#include "opt_compat_netbsd32.h" |
#include "opt_compat_ibcs2.h" |
#include "opt_compat_ibcs2.h" |
#include "opt_xen.h" |
|
|
|
#include "assym.h" |
#include "assym.h" |
#include "lapic.h" |
#include "lapic.h" |
|
|
#include <sys/errno.h> |
#include <sys/errno.h> |
#include <sys/syscall.h> |
#include <sys/syscall.h> |
|
|
|
#include <machine/param.h> |
#include <machine/pte.h> |
#include <machine/pte.h> |
|
#include <machine/pmap.h> |
#include <machine/segments.h> |
#include <machine/segments.h> |
#include <machine/specialreg.h> |
#include <machine/specialreg.h> |
#include <machine/trap.h> |
#include <machine/trap.h> |
|
|
#include <machine/i82489reg.h> |
#include <machine/i82489reg.h> |
#endif |
#endif |
|
|
/* XXX temporary kluge; these should not be here */ |
|
/* Get definitions for IOM_BEGIN, IOM_END, and IOM_SIZE */ |
|
#include <dev/isa/isareg.h> |
|
|
|
#ifdef XEN |
|
|
|
/* |
/* |
* Xen Guest Loader Info |
* override user-land alignment before including asm.h |
*/ |
*/ |
|
#define ALIGN_DATA .align 8 |
|
#define ALIGN_TEXT .align 16,0x90 |
|
#define _ALIGN_TEXT ALIGN_TEXT |
|
|
.section __xen_guest |
#include <machine/asm.h> |
|
|
|
#if defined(MULTIPROCESSOR) |
|
|
|
#define SET_CURLWP(lwp,cpu) \ |
|
movq CPUVAR(SELF),cpu ; \ |
|
movq lwp,CPUVAR(CURLWP) ; \ |
|
movq cpu,L_CPU(lwp) |
|
|
|
#else |
|
|
|
#define SET_CURLWP(lwp,tcpu) movq lwp,CPUVAR(CURLWP) |
|
#define GET_CURLWP(reg) movq CPUVAR(CURLWP),reg |
|
|
.ascii "GUEST_OS=NetBSD,GUEST_VER=4.99" |
|
.ascii ",XEN_VER=xen-3.0" |
|
.ascii ",LOADER=generic" |
|
.ascii ",VIRT_BASE=0xffffffff80000000" |
|
.ascii ",ELF_PADDR_OFFSET=0xffffffff80000000" |
|
.ascii ",VIRT_ENTRY=0xffffffff80100000" |
|
.ascii ",HYPERCALL_PAGE=0x00000101" /* (???+HYPERCALL_PAGE_OFFSET)/PAGE_SIZE) */ |
|
#if NKSYMS > 0 || defined(DDB) || defined(LKM) |
|
.ascii ",BSD_SYMTAB=yes" |
|
#endif |
#endif |
.byte 0 |
|
|
|
#endif /* XEN */ |
#define GET_CURPCB(reg) movq CPUVAR(CURPCB),reg |
|
#define SET_CURPCB(reg) movq reg,CPUVAR(CURPCB) |
|
|
|
|
|
/* XXX temporary kluge; these should not be here */ |
|
/* Get definitions for IOM_BEGIN, IOM_END, and IOM_SIZE */ |
|
#include <dev/isa/isareg.h> |
|
|
|
|
/* |
/* |
* Initialization |
* Initialization |
Line 240 _C_LABEL(lapic_isr): |
|
Line 195 _C_LABEL(lapic_isr): |
|
.space NBPG-LAPIC_ISR |
.space NBPG-LAPIC_ISR |
#endif |
#endif |
|
|
.globl _C_LABEL(cpu_id),_C_LABEL(cpu_vendorname), _C_LABEL(cpu_brand_id) |
.globl _C_LABEL(cpu_id),_C_LABEL(cpu_vendor), _C_LABEL(cpu_brand_id) |
.globl _C_LABEL(cpuid_level),_C_LABEL(cpu_feature),_C_LABEL(cpu_feature2) |
.globl _C_LABEL(cpuid_level),_C_LABEL(cpu_feature) |
.globl _C_LABEL(esym),_C_LABEL(boothowto) |
.globl _C_LABEL(esym),_C_LABEL(boothowto) |
.globl _C_LABEL(bootinfo),_C_LABEL(atdevbase) |
.globl _C_LABEL(bootinfo),_C_LABEL(atdevbase) |
.globl _C_LABEL(proc0paddr),_C_LABEL(PDPpaddr) |
.globl _C_LABEL(proc0paddr),_C_LABEL(PTDpaddr) |
.globl _C_LABEL(biosbasemem),_C_LABEL(biosextmem) |
.globl _C_LABEL(biosbasemem),_C_LABEL(biosextmem) |
.globl _C_LABEL(gdtstore) |
.globl _C_LABEL(gdtstore) |
_C_LABEL(cpu): .long 0 # are we 386, 386sx, or 486, |
_C_LABEL(cpu): .long 0 # are we 386, 386sx, or 486, |
Line 252 _C_LABEL(cpu): .long 0 # are we 386, 38 |
|
Line 207 _C_LABEL(cpu): .long 0 # are we 386, 38 |
|
_C_LABEL(cpu_id): .long 0 # saved from `cpuid' instruction |
_C_LABEL(cpu_id): .long 0 # saved from `cpuid' instruction |
_C_LABEL(cpu_feature): .long 0 # feature flags from 'cpuid' |
_C_LABEL(cpu_feature): .long 0 # feature flags from 'cpuid' |
# instruction |
# instruction |
_C_LABEL(cpu_feature2): .long 0 # feature flags from 'cpuid' |
|
# instruction |
|
_C_LABEL(cpuid_level): .long -1 # max. level accepted by 'cpuid' |
_C_LABEL(cpuid_level): .long -1 # max. level accepted by 'cpuid' |
# instruction |
# instruction |
_C_LABEL(cpu_vendorname): .space 16 # vendor string returned by `cpuid' |
_C_LABEL(cpu_vendor): .space 16 # vendor string returned by `cpuid' |
# instruction |
# instruction |
_C_LABEL(cpu_brand_id): .long 0 # brand ID from 'cpuid' instruction |
_C_LABEL(cpu_brand_id): .long 0 # brand ID from 'cpuid' instruction |
_C_LABEL(esym): .quad 0 # ptr to end of syms |
_C_LABEL(esym): .quad 0 # ptr to end of syms |
_C_LABEL(atdevbase): .quad 0 # location of start of iomem in virtual |
_C_LABEL(atdevbase): .quad 0 # location of start of iomem in virtual |
_C_LABEL(proc0paddr): .quad 0 |
_C_LABEL(proc0paddr): .quad 0 |
_C_LABEL(PDPpaddr): .quad 0 # paddr of PTD, for libkvm |
_C_LABEL(PTDpaddr): .quad 0 # paddr of PTD, for libkvm |
#ifndef REALBASEMEM |
#ifndef REALBASEMEM |
_C_LABEL(biosbasemem): .long 0 # base memory reported by BIOS |
_C_LABEL(biosbasemem): .long 0 # base memory reported by BIOS |
#else |
#else |
Line 277 _C_LABEL(biosextmem): .long REALEXTMEM |
|
Line 230 _C_LABEL(biosextmem): .long REALEXTMEM |
|
#define _RELOC(x) ((x) - KERNBASE) |
#define _RELOC(x) ((x) - KERNBASE) |
#define RELOC(x) _RELOC(_C_LABEL(x)) |
#define RELOC(x) _RELOC(_C_LABEL(x)) |
|
|
#ifndef XEN |
|
.globl gdt64 |
.globl gdt64 |
|
|
gdt64: |
gdt64: |
|
|
.long longmode-KERNBASE |
.long longmode-KERNBASE |
.word GSEL(GCODE_SEL, SEL_KPL) |
.word GSEL(GCODE_SEL, SEL_KPL) |
|
|
#endif /* !XEN */ |
|
.space 512 |
.space 512 |
tmpstk: |
tmpstk: |
|
|
|
|
.globl _C_LABEL(kernel_text) |
.globl _C_LABEL(kernel_text) |
.set _C_LABEL(kernel_text),KERNTEXTOFF |
.set _C_LABEL(kernel_text),KERNTEXTOFF |
|
|
#ifndef XEN |
|
.code32 |
.code32 |
|
|
.globl start |
.globl start |
Line 395 start: movw $0x1234,0x472 # warm boot |
|
Line 345 start: movw $0x1234,0x472 # warm boot |
|
xorl %eax,%eax |
xorl %eax,%eax |
cpuid |
cpuid |
movl %eax,RELOC(cpuid_level) |
movl %eax,RELOC(cpuid_level) |
movl $RELOC(cpu_vendorname),%ebp |
movl $RELOC(cpu_vendor),%ebp |
movl %ebx,(%ebp) |
movl %ebx,(%ebp) |
movl %edx,4(%ebp) |
movl %edx,4(%ebp) |
movl %ecx,8(%ebp) |
movl %ecx,8(%ebp) |
Line 405 start: movw $0x1234,0x472 # warm boot |
|
Line 355 start: movw $0x1234,0x472 # warm boot |
|
cpuid |
cpuid |
movl %eax,RELOC(cpu_id) |
movl %eax,RELOC(cpu_id) |
movl %edx,RELOC(cpu_feature) |
movl %edx,RELOC(cpu_feature) |
movl %ecx,RELOC(cpu_feature2) |
|
|
|
/* Brand ID is bits 0-7 of %ebx */ |
/* Brand ID is bits 0-7 of %ebx */ |
andl $255,%ebx |
andl $255,%ebx |
Line 494 start: movw $0x1234,0x472 # warm boot |
|
Line 443 start: movw $0x1234,0x472 # warm boot |
|
leal (PROC0_PTP1_OFF)(%esi), %ebx |
leal (PROC0_PTP1_OFF)(%esi), %ebx |
|
|
/* |
/* |
* Compute &__data_start - KERNBASE. This can't be > 4G, |
* Compute etext - KERNBASE. This can't be > 4G, or we can't deal |
* or we can't deal with it anyway, since we can't load it in |
* with it anyway, since we can't load it in 32 bit mode. So use |
* 32 bit mode. So use the bottom 32 bits. |
* the bottom 32 bits. |
*/ |
*/ |
movl $RELOC(__data_start),%edx |
movl $RELOC(etext),%edx |
|
addl $PGOFSET,%edx |
andl $~PGOFSET,%edx |
andl $~PGOFSET,%edx |
|
|
/* |
/* |
Line 584 start: movw $0x1234,0x472 # warm boot |
|
Line 534 start: movw $0x1234,0x472 # warm boot |
|
|
|
|
|
/* Save phys. addr of PTD, for libkvm. */ |
/* Save phys. addr of PTD, for libkvm. */ |
movl $RELOC(PDPpaddr),%ebp |
movl $RELOC(PTDpaddr),%ebp |
movl %esi,(%ebp) |
movl %esi,(%ebp) |
movl $0,4(%ebp) |
movl $0,4(%ebp) |
|
|
|
|
|
|
/* XXX merge these */ |
/* XXX merge these */ |
leaq TABLESIZE(%rsi),%rdi |
leaq TABLESIZE(%rsi),%rdi |
|
|
#else /* XEN */ |
|
.globl start |
|
start: |
|
/* First, reset the PSL. */ |
|
pushq $2 |
|
popfq |
|
|
|
cld |
|
|
|
/* |
|
* Xen info: |
|
* - %rsi -> start_info struct |
|
* - %rsp -> stack, *theorically* the last used page |
|
* by Xen bootstrap |
|
*/ |
|
movq %rsi, %rbx |
|
|
|
/* Clear BSS */ |
|
xorq %rax,%rax |
|
movq $_C_LABEL(__bss_start),%rdi |
|
movq $_C_LABEL(_end),%rcx |
|
subq %rdi,%rcx |
|
rep |
|
stosb |
|
|
|
/* Copy start_info to a safe place */ |
|
movq %rbx,%rsi |
|
movq $_C_LABEL(start_info_union),%rdi |
|
movq $64,%rcx |
|
rep |
|
movsq |
|
|
|
/* |
|
* Memory layout at start of the day: |
|
* - Kernel image |
|
* - Page frames list |
|
* - start_info struct. we copied it, so it can be recycled. |
|
* - xenstore |
|
* - console |
|
* - Xen bootstrap page tables |
|
* - kernel stack. provided by Xen |
|
* - guaranted 512kB padding |
|
* |
|
* As we want to rebuild our page tables and place our stack |
|
* in proc0 struct, all data starting from after console can be |
|
* discarded after we've done a little setup. |
|
*/ |
|
|
|
/* |
|
* We want our own page tables, let's rebuild them |
|
* We will reclaim xen space afterward INCLUDING stack |
|
* so let's change it to a temporary one |
|
*/ |
|
|
|
movq $tmpstk, %rax |
|
subq $8, %rax |
|
movq %rax, %rsp |
|
|
|
call xen_pmap_bootstrap |
|
|
|
/* |
|
* First avail returned by xen_pmap_bootstrap in %rax |
|
*/ |
|
movq %rax, %rsi |
|
movq %rsi,_C_LABEL(proc0paddr) |
|
|
|
xorq %rax,%rax |
|
movq %rsi,%rdi |
|
movq $USPACE,%rcx |
|
rep |
|
stosb |
|
|
|
/* |
|
* Set new stack and clear segments |
|
*/ |
|
|
|
leaq (USPACE-FRAMESIZE)(%rsi),%rsp |
|
xorq %rbp,%rbp |
|
|
|
movw %ax,%gs |
|
movw %ax,%fs |
|
|
|
/* |
|
* Set first_avail after proc0 |
|
*/ |
|
|
|
movq %rsi,%rdi |
|
addq $USPACE,%rdi |
|
subq $KERNBASE,%rdi # init_x86_64 want a physical address |
|
|
|
#endif /* !XEN */ |
|
call _C_LABEL(init_x86_64) |
call _C_LABEL(init_x86_64) |
|
|
call _C_LABEL(main) |
call _C_LABEL(main) |
|
|
#ifdef XEN |
/*****************************************************************************/ |
/* space for the hypercall call page */ |
|
#define HYPERCALL_PAGE_OFFSET 0x1000 |
|
.org HYPERCALL_PAGE_OFFSET |
|
ENTRY(hypercall_page) |
|
.skip 0x1000 |
|
#endif /* XEN */ |
|
|
|
/* |
/* |
* int setjmp(label_t *) |
* void lgdt(struct region_descriptor *rdp); |
* |
* Change the global descriptor table. |
* Used primarily by DDB. |
|
*/ |
*/ |
|
NENTRY(lgdt) |
|
/* Reload the descriptor table. */ |
|
movq %rdi,%rax |
|
lgdt (%rax) |
|
/* Flush the prefetch q. */ |
|
jmp 1f |
|
nop |
|
1: /* Reload "stale" selectors. */ |
|
movl $GSEL(GDATA_SEL, SEL_KPL),%eax |
|
movl %eax,%ds |
|
movl %eax,%es |
|
movl %eax,%ss |
|
/* Reload code selector by doing intersegment return. */ |
|
popq %rax |
|
pushq $GSEL(GCODE_SEL, SEL_KPL) |
|
pushq %rax |
|
lretq |
|
|
ENTRY(setjmp) |
ENTRY(setjmp) |
/* |
/* |
* Only save registers that must be preserved across function |
* Only save registers that must be preserved across function |
|
|
xorl %eax,%eax |
xorl %eax,%eax |
ret |
ret |
|
|
/* |
|
* int longjmp(label_t *) |
|
* |
|
* Used primarily by DDB. |
|
*/ |
|
ENTRY(longjmp) |
ENTRY(longjmp) |
movq %rdi,%rax |
movq %rdi,%rax |
movq (%rax),%rbx |
movq (%rax),%rbx |
|
|
movq 48(%rax),%r15 |
movq 48(%rax),%r15 |
movq 56(%rax),%rdx |
movq 56(%rax),%rdx |
movq %rdx,(%rsp) |
movq %rdx,(%rsp) |
movl $1,%eax |
xorl %eax,%eax |
|
incl %eax |
ret |
ret |
|
|
ENTRY(dumpsys) |
/*****************************************************************************/ |
# mimic cpu_switchto() for postmortem debugging. |
|
|
|
# build a fake switch frame. |
/* |
pushq %rbx |
* The following primitives manipulate the run queues. |
pushq %r12 |
* _whichqs tells which of the 32 queues _qs |
pushq %r13 |
* have processes in them. Setrq puts processes into queues, Remrq |
pushq %r14 |
* removes them from queues. The running process is on no queue, |
pushq %r15 |
* other processes are on a queue related to p->p_pri, divided by 4 |
|
* actually to shrink the 0-127 range of priorities into the 32 available |
# save a context. |
* queues. |
movq $dumppcb, %rax |
*/ |
movq %rsp, PCB_RSP(%rax) |
.globl _C_LABEL(sched_whichqs),_C_LABEL(sched_qs) |
movq %rbp, PCB_RBP(%rax) |
.globl _C_LABEL(uvmexp),_C_LABEL(panic) |
|
|
call _C_LABEL(dodumpsys) |
#if NAPM > 0 |
|
.globl _C_LABEL(apm_cpu_idle),_C_LABEL(apm_cpu_busy) |
|
#endif |
|
|
addq $(5*8), %rsp # sizeof(switchframe) - sizeof(%rip) |
#ifdef DIAGNOSTIC |
ret |
NENTRY(switch_error1) |
|
movabsq $1f,%rdi |
|
call _C_LABEL(panic) |
|
/* NOTREACHED */ |
|
1: .asciz "cpu_switch 1" |
|
NENTRY(switch_error2) |
|
movabsq $1f,%rdi |
|
call _C_LABEL(panic) |
|
/* NOTREACHED */ |
|
1: .asciz "cpu_switch 2" |
|
NENTRY(switch_error3) |
|
movabsq $1f,%rdi |
|
call _C_LABEL(panic) |
|
/* NOTREACHED */ |
|
1: .asciz "cpu_switch 3" |
|
#endif /* DIAGNOSTIC */ |
|
|
/* |
/* |
* struct lwp *cpu_switchto(struct lwp *oldlwp, struct newlwp, |
* int cpu_switch(struct proc *) |
* bool returning) |
* Find a runnable process and switch to it. Wait if necessary. If the new |
* |
* lwp is the same as the old one, we short-circuit the context save and |
* 1. if (oldlwp != NULL), save its context. |
* restore. |
* 2. then, restore context of newlwp. |
|
* |
|
* Note that the stack frame layout is known to "struct switchframe" in |
|
* <machine/frame.h> and to the code in cpu_lwp_fork() which initializes |
|
* it for a new lwp. |
|
*/ |
*/ |
ENTRY(cpu_switchto) |
ENTRY(cpu_switch) |
pushq %rbx |
pushq %rbx |
|
pushq %rbp |
pushq %r12 |
pushq %r12 |
pushq %r13 |
pushq %r13 |
pushq %r14 |
pushq %r14 |
pushq %r15 |
pushq %r15 |
|
|
movq %rdi,%r13 # oldlwp |
movq %rdi,%r13 |
movq %rsi,%r12 # newlwp |
|
|
|
testq %r13,%r13 |
/* |
jz 1f |
* Clear curlwp so that we don't accumulate system time while idle. |
|
* This also insures that schedcpu() will move the old lwp to |
|
* the correct queue if it happens to get called from the spllower() |
|
* below and changes the priority. (See corresponding comment in |
|
* userret()). |
|
*/ |
|
movq $0,CPUVAR(CURLWP) |
|
|
/* Save old context. */ |
|
movq L_ADDR(%r13),%rax |
|
movq %rsp,PCB_RSP(%rax) |
|
movq %rbp,PCB_RBP(%rax) |
|
|
|
/* Switch to newlwp's stack. */ |
|
1: movq L_ADDR(%r12),%r14 |
|
#ifdef XEN /* XXX debug code */ |
|
cmpq $0, PCB_RSP(%r14) |
|
jne 999f |
|
callq _C_LABEL(cpu_Debugger); |
|
999: |
|
#endif |
|
movq PCB_RSP(%r14),%rsp |
|
movq PCB_RBP(%r14),%rbp |
|
|
|
/* |
/* |
* Set curlwp. This must be globally visible in order to permit |
* First phase: find new lwp. |
* non-interlocked mutex release. |
* |
|
* Registers: |
|
* %rax - queue head, scratch, then zero |
|
* %r8 - queue number |
|
* %ecx - cached value of whichqs |
|
* %rdx - next process in queue |
|
* %r13 - old lwp |
|
* %r12 - new lwp |
|
*/ |
|
|
|
/* Look for new lwp. */ |
|
cli # splhigh doesn't do a cli |
|
movl _C_LABEL(sched_whichqs)(%rip),%ecx |
|
bsfl %ecx,%r8d # find a full q |
|
jnz switch_dequeue |
|
|
|
/* |
|
* idling: save old context |
|
* |
|
* Registers: |
|
* %rax, %rcx - scratch |
|
* %r13 - old lwp, then old pcb |
|
* %r12 - idle pcb |
*/ |
*/ |
movq %r12,%rcx |
|
xchgq %rcx,CPUVAR(CURLWP) |
|
|
|
/* Skip the rest if returning to a pinned LWP. */ |
/* old LWP still in %rdi */ |
testb %dl,%dl |
call _C_LABEL(pmap_deactivate) |
jnz 4f |
|
|
movq L_ADDR(%r13),%r13 |
/* Switch ring0 stack */ |
|
#ifndef XEN |
/* Save stack pointers */ |
movq PCB_RSP0(%r14),%rax |
|
movq %rax,CPUVAR(RSP0) |
movq %rsp,PCB_RSP(%r13) |
|
movq %rbp,PCB_RBP(%r13) |
|
|
|
/* Find idle PCB for this CPU */ |
|
#ifndef MULTIPROCESSOR |
|
leaq _C_LABEL(lwp0)(%rip),%rcx |
|
movq L_ADDR(%rcx),%r12 |
|
movl L_MD_TSS_SEL(%rcx),%edx |
#else |
#else |
movq %r14, %rdi |
movq CPUVAR(IDLE_PCB),%r12 |
callq _C_LABEL(x86_64_switch_context); |
movl CPUVAR(IDLE_TSS_SEL),%edx |
#endif |
#endif |
|
movq $0,CPUVAR(CURLWP) |
|
|
/* Don't bother with the rest if switching to a system process. */ |
/* Restore the idle context (avoid interrupts) */ |
testl $LW_SYSTEM,L_FLAG(%r12) |
cli |
jnz 4f |
|
|
|
/* Is this process using RAS (restartable atomic sequences)? */ |
/* Restore stack pointers. */ |
movq L_PROC(%r12),%rdi |
movq PCB_RSP(%r12),%rsp |
cmpq $0,P_RASLIST(%rdi) |
movq PCB_RBP(%r12),%rbp |
jne 5f |
|
|
/* Switch address space. */ |
|
movq PCB_CR3(%r12),%rcx |
|
movq %rcx,%cr3 |
|
|
|
#ifdef MULTIPROCESSOR |
|
movq CPUVAR(GDT),%rax |
|
#else |
|
movq _C_LABEL(gdtstore)(%rip),%rax |
|
#endif |
|
|
|
/* Switch TSS. Reset "task busy" flag before */ |
|
andl $~0x0200,4(%rax,%rdx, 1) |
|
ltr %dx |
|
|
|
/* Restore cr0 (including FPU state). */ |
|
movl PCB_CR0(%r12),%ecx |
|
movq %rcx,%cr0 |
|
|
|
SET_CURPCB(%r12) |
|
|
|
xorq %r13,%r13 |
|
sti |
|
idle_unlock: |
|
#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) |
|
call _C_LABEL(sched_unlock_idle) |
|
#endif |
|
/* Interrupts are okay again. */ |
|
movl $IPL_NONE,%edi |
|
call _C_LABEL(Xspllower) |
|
jmp idle_start |
|
idle_zero: |
|
sti |
|
call _C_LABEL(uvm_pageidlezero) |
|
cli |
|
cmpl $0,_C_LABEL(sched_whichqs)(%rip) |
|
jnz idle_exit |
|
idle_loop: |
|
/* Try to zero some pages. */ |
|
movl _C_LABEL(uvm)+UVM_PAGE_IDLE_ZERO(%rip),%ecx |
|
testl %ecx,%ecx |
|
jnz idle_zero |
|
sti |
|
hlt |
|
NENTRY(mpidle) |
|
idle_start: |
|
cli |
|
cmpl $0,_C_LABEL(sched_whichqs)(%rip) |
|
jz idle_loop |
|
idle_exit: |
|
movl $IPL_HIGH,CPUVAR(ILEVEL) |
|
sti |
|
#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) |
|
call _C_LABEL(sched_lock_idle) |
|
#endif |
|
movl _C_LABEL(sched_whichqs)(%rip),%ecx |
|
bsfl %ecx,%r8d |
|
jz idle_unlock |
|
|
|
switch_dequeue: |
|
|
|
sti |
|
movq %r8,%r9 |
|
|
|
shlq $4, %r9 |
|
leaq _C_LABEL(sched_qs)(%rip),%rax |
|
addq %r9,%rax |
|
/* movq (%rax),%rax */ |
|
|
|
movq L_FORW(%rax),%r12 # unlink from front of process q |
|
#ifdef DIAGNOSTIC |
|
cmpq %r12,%rax # linked to self (i.e. nothing queued)? |
|
je _C_LABEL(switch_error1) # not possible |
|
#endif /* DIAGNOSTIC */ |
|
movq L_FORW(%r12),%rdx |
|
movq %rdx,L_FORW(%rax) |
|
movq %rax,L_BACK(%rdx) |
|
|
|
cmpq %rdx,%rax # q empty? |
|
jne 3f |
|
|
|
btrl %r8d,%ecx # yes, clear to indicate empty |
|
movl %ecx,_C_LABEL(sched_whichqs)(%rip) # update q status |
|
|
|
3: /* We just did it. */ |
|
xorq %rax,%rax |
|
movl %eax,CPUVAR(RESCHED) |
|
switch_resume: |
|
#ifdef DIAGNOSTIC |
|
cmpq %rax,L_WCHAN(%r12) |
|
jne _C_LABEL(switch_error2) |
|
cmpb $LSRUN,L_STAT(%r12) |
|
jne _C_LABEL(switch_error3) |
|
#endif |
|
|
|
/* Isolate lwp. XXX Is this necessary? */ |
|
movq %rax,L_BACK(%r12) |
|
|
|
/* Record new lwp. */ |
|
movb $LSONPROC,L_STAT(%r12) # l->l_stat = SONPROC |
|
SET_CURLWP(%r12,%rcx) |
|
|
|
/* Skip context switch if same lwp. */ |
|
xorl %ebx,%ebx |
|
cmpq %r12,%r13 |
|
je switch_return |
|
|
|
/* If old lwp exited, don't bother. */ |
|
testq %r13,%r13 |
|
jz switch_exited |
|
|
/* |
/* |
* Restore cr0 (including FPU state). Raise the IPL to IPL_IPI. |
* Second phase: save old context. |
* FPU IPIs can alter the LWP's saved cr0. Dropping the priority |
* |
* is deferred until mi_switch(), when cpu_switchto() returns. |
* Registers: |
|
* %rax, %rcx - scratch |
|
* %r13 - old lwp, then old pcb |
|
* %r12 - new lwp |
*/ |
*/ |
2: |
|
#ifndef XEN |
|
movl $IPL_IPI,CPUVAR(ILEVEL) |
|
movl PCB_CR0(%r14),%ecx |
|
movq %cr0,%rdx |
|
|
|
|
movq %r13,%rdi |
|
call pmap_deactivate |
|
|
|
movq L_ADDR(%r13),%r13 |
|
|
|
/* Save stack pointers. */ |
|
movq %rsp,PCB_RSP(%r13) |
|
movq %rbp,PCB_RBP(%r13) |
|
|
|
switch_exited: |
/* |
/* |
* If our floating point registers are on a different CPU, |
* Third phase: restore saved context. |
* set CR0_TS so we'll trap rather than reuse bogus state. |
* |
|
* Registers: |
|
* %rax, %rcx, %rdx - scratch |
|
* %r13 - new pcb |
|
* %r12 - new process |
*/ |
*/ |
movq PCB_FPCPU(%r14),%rax |
|
cmpq CPUVAR(SELF),%rax |
/* No interrupts while loading new state. */ |
je 3f |
cli |
orq $CR0_TS,%rcx |
movq L_ADDR(%r12),%r13 |
|
|
/* Reloading CR0 is very expensive - avoid if possible. */ |
/* Restore stack pointers. */ |
3: cmpq %rdx,%rcx |
movq PCB_RSP(%r13),%rsp |
je 4f |
movq PCB_RBP(%r13),%rbp |
|
|
|
#if 0 |
|
/* Don't bother with the rest if switching to a system process. */ |
|
testl $P_SYSTEM,P_FLAG(%r12) |
|
jnz switch_restored |
|
#endif |
|
|
|
/* Load TSS info. */ |
|
#ifdef MULTIPROCESSOR |
|
movq CPUVAR(GDT),%rax |
|
#else |
|
movq _C_LABEL(gdtstore)(%rip),%rax |
|
#endif |
|
movl L_MD_TSS_SEL(%r12),%edx |
|
|
|
/* Switch TSS. Reset "task busy" flag before */ |
|
andl $~0x0200,4(%rax,%rdx, 1) |
|
ltr %dx |
|
|
|
movq %r12,%rdi |
|
call _C_LABEL(pmap_activate) |
|
|
|
#if 0 |
|
switch_restored: |
|
#endif |
|
/* Restore cr0 (including FPU state). */ |
|
movl PCB_CR0(%r13),%ecx |
|
#ifdef MULTIPROCESSOR |
|
movq PCB_FPCPU(%r13),%r8 |
|
cmpq CPUVAR(SELF),%r8 |
|
jz 1f |
|
orl $CR0_TS,%ecx |
|
1: |
|
#endif |
movq %rcx,%cr0 |
movq %rcx,%cr0 |
|
|
|
SET_CURPCB(%r13) |
|
|
|
/* Interrupts are okay again. */ |
|
sti |
|
|
|
/* |
|
* Check for restartable atomic sequences (RAS) |
|
*/ |
|
movq CPUVAR(CURLWP),%r12 |
|
movq L_PROC(%r12),%rdi |
|
cmpq $0,P_RASLIST(%rdi) |
|
je 1f |
|
movq L_MD_REGS(%r12),%rbx |
|
movq TF_RIP(%rbx),%rsi |
|
call _C_LABEL(ras_lookup) |
|
cmpq $-1,%rax |
|
je 1f |
|
movq %rax,TF_RIP(%rbx) |
|
1: |
|
movl $1,%ebx |
|
|
|
switch_return: |
|
#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) |
|
call _C_LABEL(sched_unlock_idle) |
#endif |
#endif |
|
|
/* Return to the new LWP, returning 'oldlwp' in %rax. */ |
movl $IPL_NONE,%edi |
4: movq %r13,%rax |
call _C_LABEL(Xspllower) |
|
movl $IPL_HIGH,CPUVAR(ILEVEL) |
|
|
|
movl %ebx,%eax |
|
|
popq %r15 |
popq %r15 |
popq %r14 |
popq %r14 |
popq %r13 |
popq %r13 |
popq %r12 |
popq %r12 |
|
popq %rbp |
popq %rbx |
popq %rbx |
ret |
ret |
|
|
/* Check for restartable atomic sequences (RAS). */ |
ENTRY(cpu_switchto) |
5: movq L_MD_REGS(%r12),%rbx |
pushq %rbx |
movq TF_RIP(%rbx),%rsi |
pushq %rbp |
call _C_LABEL(ras_lookup) |
pushq %r12 |
cmpq $-1,%rax |
pushq %r13 |
je 2b |
pushq %r14 |
movq %rax,TF_RIP(%rbx) |
pushq %r15 |
jmp 2b |
|
|
movq %rdi,%r13 |
|
movq %rsi,%r12 |
|
|
|
movq $0,CPUVAR(CURLWP) |
|
|
|
xorq %rax,%rax |
|
jmp switch_resume |
|
|
|
|
/* |
/* |
* void savectx(struct pcb *pcb); |
* void switch_exit(struct lwp *l, void (*exit)(struct lwp *)); |
* |
* Switch to proc0's saved context and deallocate the address space and kernel |
|
* stack for p. Then jump into cpu_switch(), as if we were in proc0 all along. |
|
*/ |
|
.globl _C_LABEL(lwp0) |
|
ENTRY(switch_exit) |
|
#ifdef MULTIPROCESSOR |
|
movq CPUVAR(IDLE_PCB),%r8 |
|
movl CPUVAR(IDLE_TSS_SEL),%edx |
|
#else |
|
leaq _C_LABEL(lwp0)(%rip),%r9 |
|
movq L_ADDR(%r9),%r8 |
|
movl L_MD_TSS_SEL(%r9),%edx |
|
#endif |
|
|
|
/* In case we fault... */ |
|
movq $0,CPUVAR(CURLWP) |
|
|
|
cli |
|
|
|
/* Restore stack pointers. */ |
|
movq PCB_RSP(%r8),%rsp |
|
movq PCB_RBP(%r8),%rbp |
|
|
|
/* Load TSS info. */ |
|
#ifdef MULTIPROCESSOR |
|
movq CPUVAR(GDT),%rax |
|
#else |
|
movq _C_LABEL(gdtstore)(%rip),%rax |
|
#endif |
|
|
|
/* Switch address space. */ |
|
movq PCB_CR3(%r8),%rcx |
|
movq %rcx,%cr3 |
|
|
|
/* Switch TSS. */ |
|
andl $~0x0200,4-SEL_KPL(%rax,%rdx,1) |
|
ltr %dx |
|
|
|
/* We're always in the kernel, so we don't need the LDT. */ |
|
|
|
/* Restore cr0 (including FPU state). */ |
|
movl PCB_CR0(%r8),%ecx |
|
movq %rcx,%cr0 |
|
|
|
/* Record new pcb. */ |
|
SET_CURPCB(%r8) |
|
|
|
/* Interrupts are okay again. */ |
|
sti |
|
|
|
/* |
|
* Schedule the dead process's vmspace and stack to be freed. |
|
* {lpw_}exit2(l). Function still in %rsi (2nd arg), lwp in |
|
* %rdi (first arg). |
|
*/ |
|
|
|
call *%rsi |
|
|
|
/* Jump into cpu_switch() with the right state. */ |
|
xorq %r13,%r13 |
|
movq %r13, CPUVAR(CURLWP) |
|
jmp idle_start |
|
|
|
/* |
|
* savectx(struct pcb *pcb); |
* Update pcb, saving current processor state. |
* Update pcb, saving current processor state. |
*/ |
*/ |
ENTRY(savectx) |
ENTRY(savectx) |
/* Save stack pointers. */ |
/* Save stack pointers. */ |
movq %rsp,PCB_RSP(%rdi) |
movq %rsp,PCB_RSP(%rdi) |
movq %rbp,PCB_RBP(%rdi) |
movq %rbp,PCB_RBP(%rdi) |
|
|
ret |
ret |
|
|
IDTVEC(syscall32) |
IDTVEC(syscall32) |
sysret /* go away please */ |
sysret /* go away please */ |
|
|
/* |
/* |
* syscall() |
|
* |
|
* syscall insn entry. This currently isn't much faster, but |
* syscall insn entry. This currently isn't much faster, but |
* it can be made faster in the future. |
* it can be made faster in the future. |
*/ |
*/ |
IDTVEC(syscall) |
IDTVEC(syscall) |
#ifndef XEN |
|
swapgs |
swapgs |
movq %r15,CPUVAR(SCRATCH) |
movq %r15,CPUVAR(SCRATCH) |
movq CPUVAR(CURLWP),%r15 |
movq CPUVAR(CURPCB),%r15 |
movq L_ADDR(%r15),%r15 |
|
movq PCB_RSP0(%r15),%r15 |
movq PCB_RSP0(%r15),%r15 |
xchgq %r15,%rsp |
xchgq %r15,%rsp |
sti |
sti |
Line 1014 IDTVEC(syscall) |
|
Line 1162 IDTVEC(syscall) |
|
* ss:rsp, etc, so that all GP registers can be |
* ss:rsp, etc, so that all GP registers can be |
* saved. Then, fill in the rest. |
* saved. Then, fill in the rest. |
*/ |
*/ |
pushq $(LSEL(LUDATA_SEL, SEL_UPL)) /* Known to be user ss */ |
pushq $(LSEL(LUDATA_SEL, SEL_UPL)) |
pushq %r15 /* User space rsp */ |
pushq %r15 |
|
subq $(TF_RSP-TF_TRAPNO),%rsp |
movq CPUVAR(SCRATCH),%r15 |
movq CPUVAR(SCRATCH),%r15 |
subq $TF_REGSIZE+(TF_RSP-TF_TRAPNO),%rsp |
subq $32,%rsp |
INTR_SAVE_GPRS |
INTR_SAVE_GPRS |
movw %fs,TF_FS(%rsp) |
movw %fs,TF_FS(%rsp) |
movw %gs,TF_GS(%rsp) |
movw %gs,TF_GS(%rsp) |
Line 1025 IDTVEC(syscall) |
|
Line 1174 IDTVEC(syscall) |
|
movw $(LSEL(LUDATA_SEL, SEL_UPL)),TF_DS(%rsp) |
movw $(LSEL(LUDATA_SEL, SEL_UPL)),TF_DS(%rsp) |
movq %r11, TF_RFLAGS(%rsp) /* old rflags from syscall insn */ |
movq %r11, TF_RFLAGS(%rsp) /* old rflags from syscall insn */ |
movq $(LSEL(LUCODE_SEL, SEL_UPL)), TF_CS(%rsp) |
movq $(LSEL(LUCODE_SEL, SEL_UPL)), TF_CS(%rsp) |
movq %rcx,TF_RIP(%rsp) /* syscall saves rip in rcx */ |
movq %rcx,TF_RIP(%rsp) |
movq $2,TF_ERR(%rsp) |
movq $2,TF_ERR(%rsp) |
movq $T_ASTFLT, TF_TRAPNO(%rsp) |
movq $T_ASTFLT, TF_TRAPNO(%rsp) |
#else |
|
/* Xen already switched to kernel stack */ |
|
pushq %rsi |
|
STI(si) |
|
popq %rsi |
|
addq $0x10,%rsp /* gap to match cs:rip */ |
|
pushq $2 /* error code */ |
|
pushq $T_ASTFLT |
|
subq $TF_REGSIZE,%rsp |
|
INTR_SAVE_GPRS |
|
movw %fs,TF_FS(%rsp) |
|
movw %gs,TF_GS(%rsp) |
|
movw %es,TF_ES(%rsp) |
|
movw $(LSEL(LUDATA_SEL, SEL_UPL)),TF_DS(%rsp) |
|
#endif |
|
|
|
movq CPUVAR(CURLWP),%r14 |
movq CPUVAR(CURLWP),%r14 |
incl CPUVAR(NSYSCALL) # count it atomically |
|
movq %rsp,L_MD_REGS(%r14) # save pointer to frame |
movq %rsp,L_MD_REGS(%r14) # save pointer to frame |
movq L_PROC(%r14),%r15 |
movq L_PROC(%r14),%r15 |
andl $~MDP_IRET,L_MD_FLAGS(%r14) |
andl $~MDP_IRET,L_MD_FLAGS(%r14) |
movq %rsp,%rdi /* Pass frame as arg0 */ |
movq %rsp,%rdi |
call *P_MD_SYSCALL(%r15) |
call *P_MD_SYSCALL(%r15) |
.Lsyscall_checkast: |
1: /* Check for ASTs on exit to user mode. */ |
/* Check for ASTs on exit to user mode. */ |
cli |
CLI(si) |
CHECK_ASTPENDING(%r11) |
CHECK_ASTPENDING(%r14) |
je 2f |
je 1f |
|
/* Always returning to user mode here. */ |
/* Always returning to user mode here. */ |
CLEAR_ASTPENDING(%r14) |
CLEAR_ASTPENDING(%r11) |
STI(si) |
sti |
/* Pushed T_ASTFLT into tf_trapno on entry. */ |
/* Pushed T_ASTFLT into tf_trapno on entry. */ |
movq %rsp,%rdi |
movq %rsp,%rdi |
call _C_LABEL(trap) |
call _C_LABEL(trap) |
jmp .Lsyscall_checkast /* re-check ASTs */ |
jmp 1b |
1: CHECK_DEFERRED_SWITCH |
2: |
jnz 9f |
sti |
STI(si) |
|
testl $MDP_IRET, L_MD_FLAGS(%r14) |
testl $MDP_IRET, L_MD_FLAGS(%r14) |
jne iret_return; |
jne iret_return; |
syscall_return: |
syscall_return: |
Line 1076 syscall_return: |
|
Line 1207 syscall_return: |
|
/* |
/* |
* XXX interrupts off longer than they should be here. |
* XXX interrupts off longer than they should be here. |
*/ |
*/ |
#ifndef XEN |
|
cli |
cli |
swapgs |
swapgs |
#endif |
|
movw TF_ES(%rsp),%es |
movw TF_ES(%rsp),%es |
movw TF_FS(%rsp),%fs |
movw TF_FS(%rsp),%fs |
#ifndef XEN |
|
movw TF_GS(%rsp),%gs |
movw TF_GS(%rsp),%gs |
#endif |
|
INTR_RESTORE_GPRS |
INTR_RESTORE_GPRS |
movw $(LSEL(LUDATA_SEL, SEL_UPL)),%r11 |
movw $(LSEL(LUDATA_SEL, SEL_UPL)),%r11 |
movw %r11,%ds |
movw %r11,%ds |
addq $TF_REGSIZE+16,%rsp /* + T_xxx and error code */ |
addq $48,%rsp |
#ifndef XEN |
|
popq %rcx /* return rip */ |
popq %rcx /* return rip */ |
addq $8,%rsp /* discard cs */ |
addq $8,%rsp |
popq %r11 /* flags as set by sysret insn */ |
popq %r11 /* flags as set by sysret insn */ |
movq %ss:(%rsp),%rsp |
movq %ss:(%rsp),%rsp |
sysretq |
sysretq |
#else |
|
pushq $256 /* VGCF_IN_SYSCALL */ |
|
jmp HYPERVISOR_iret |
|
#endif |
|
|
|
#ifdef DIAGNOSTIC |
#ifdef DIAGNOSTIC |
3: movabsq $4f, %rdi |
3: movabsq $4f, %rdi |
Line 1108 syscall_return: |
|
Line 1230 syscall_return: |
|
movl CPUVAR(ILEVEL),%r8d |
movl CPUVAR(ILEVEL),%r8d |
xorq %rax,%rax |
xorq %rax,%rax |
call _C_LABEL(printf) |
call _C_LABEL(printf) |
movl $IPL_NONE,%edi |
#ifdef DDB |
call _C_LABEL(spllower) |
int $3 |
jmp .Lsyscall_checkast |
#endif /* DDB */ |
|
movl $IPL_NONE,CPUVAR(ILEVEL) |
|
jmp 1b |
4: .asciz "WARNING: SPL NOT LOWERED ON SYSCALL %d %d EXIT %x %x\n" |
4: .asciz "WARNING: SPL NOT LOWERED ON SYSCALL %d %d EXIT %x %x\n" |
#endif |
#endif |
|
|
9: STI(si) |
|
call _C_LABEL(do_pmap_load) |
|
jmp .Lsyscall_checkast /* re-check ASTs */ |
|
|
|
/* |
NENTRY(proc_trampoline) |
* void lwp_trampoline(void); |
#ifdef MULTIPROCESSOR |
* |
call _C_LABEL(proc_trampoline_mp) |
* This is a trampoline function pushed run by newly created LWPs |
#endif |
* in order to do additional setup in their context. 32-bit |
movl $IPL_NONE,CPUVAR(ILEVEL) |
* binaries begin life here. |
|
*/ |
|
NENTRY(lwp_trampoline) |
|
movq %rbp,%rsi |
|
movq %rax,%rdi |
|
xorq %rbp,%rbp |
|
call _C_LABEL(lwp_startup) |
|
movq %r13,%rdi |
movq %r13,%rdi |
call *%r12 |
call *%r12 |
DO_DEFERRED_SWITCH |
|
INTRFASTEXIT |
INTRFASTEXIT |
/* NOTREACHED */ |
/* NOTREACHED */ |
|
|
/* |
|
* void child_trampoline(void); |
|
* |
|
* As per lwp_trampoline(), but 64-bit binaries start here. |
|
*/ |
|
NENTRY(child_trampoline) |
NENTRY(child_trampoline) |
movq %rbp,%rsi |
#ifdef MULTIPROCESSOR |
movq %rax,%rdi |
call _C_LABEL(proc_trampoline_mp) |
xorq %rbp,%rbp |
#endif |
call _C_LABEL(lwp_startup) |
movl $IPL_NONE,CPUVAR(ILEVEL) |
movq %r13,%rdi |
movq %r13,%rdi |
call *%r12 |
call *%r12 |
DO_DEFERRED_SWITCH |
|
jmp syscall_return |
jmp syscall_return |
|
|
|
#if defined(COMPAT_16) || defined(COMPAT_IBCS2) || defined(COMPAT_NETBSD32) |
|
|
.globl _C_LABEL(osyscall_return) |
.globl _C_LABEL(osyscall_return) |
|
|
|
#if defined(COMPAT_10) || defined(COMPAT_IBCS2) |
/* |
/* |
* oosyscall() |
|
* |
|
* Old call gate entry for syscall. only needed if we're |
* Old call gate entry for syscall. only needed if we're |
* going to support running old i386 NetBSD 1.0 or ibcs2 binaries, etc, |
* going to support running old NetBSD or ibcs2 binaries, etc, |
* on NetBSD/amd64. |
* on NetBSD/amd64. |
* The 64bit call gate can't request that arguments be copied from the |
|
* user stack (which the i386 code uses to get a gap for the flags). |
|
* push/pop are <read>:<modify_sp>:<write> cycles. |
|
*/ |
*/ |
IDTVEC(oosyscall) |
IDTVEC(oosyscall) |
/* Set rflags in trap frame. */ |
/* Set rflags in trap frame. */ |
pushq (%rsp) # move user's %eip |
|
pushq 16(%rsp) # and %cs |
|
popq 8(%rsp) |
|
pushfq |
pushfq |
popq 16(%rsp) |
popq 8(%rsp) |
pushq $7 # size of instruction for restart |
pushq $7 # size of instruction for restart |
jmp osyscall1 |
jmp osyscall1 |
|
|
|
#endif |
|
|
/* |
/* |
* osyscall() |
|
* |
|
* Trap gate entry for int $80 syscall, also used by sigreturn. |
* Trap gate entry for int $80 syscall, also used by sigreturn. |
*/ |
*/ |
IDTVEC(osyscall) |
IDTVEC(osyscall) |
#ifdef XEN |
|
movq (%rsp),%rcx |
|
movq 8(%rsp),%r11 |
|
addq $0x10,%rsp |
|
#endif |
|
pushq $2 # size of instruction for restart |
pushq $2 # size of instruction for restart |
osyscall1: |
osyscall1: |
pushq $T_ASTFLT # trap # for doing ASTs |
pushq $T_ASTFLT # trap # for doing ASTs |
INTRENTRY |
INTRENTRY |
STI(si) |
sti |
movq CPUVAR(CURLWP),%r14 |
movq CPUVAR(CURLWP),%rdx |
movq %rsp,L_MD_REGS(%r14) # save pointer to frame |
movq %rsp,L_MD_REGS(%rdx) # save pointer to frame |
movq L_PROC(%r14),%rdx |
movq L_PROC(%rdx),%rdx |
movq %rsp,%rdi |
movq %rsp,%rdi |
call *P_MD_SYSCALL(%rdx) |
call *P_MD_SYSCALL(%rdx) |
_C_LABEL(osyscall_return): |
_C_LABEL(osyscall_return): |
.Losyscall_checkast: |
2: /* Check for ASTs on exit to user mode. */ |
/* Check for ASTs on exit to user mode. */ |
cli |
CLI(si) |
CHECK_ASTPENDING(%r11) |
CHECK_ASTPENDING(%r14) |
|
je 1f |
je 1f |
/* Always returning to user mode here. */ |
/* Always returning to user mode here. */ |
CLEAR_ASTPENDING(%r14) |
CLEAR_ASTPENDING(%r11) |
STI(si) |
sti |
/* Pushed T_ASTFLT into tf_trapno on entry. */ |
/* Pushed T_ASTFLT into tf_trapno on entry. */ |
movq %rsp,%rdi |
movq %rsp,%rdi |
call _C_LABEL(trap) |
call _C_LABEL(trap) |
jmp .Losyscall_checkast |
jmp 2b |
1: CHECK_DEFERRED_SWITCH |
|
jnz 9f |
#endif |
|
|
iret_return: |
iret_return: |
#ifndef DIAGNOSTIC |
#ifndef DIAGNOSTIC |
INTRFASTEXIT |
1: INTRFASTEXIT |
#else /* DIAGNOSTIC */ |
#else /* DIAGNOSTIC */ |
cmpl $IPL_NONE,CPUVAR(ILEVEL) |
1: cmpl $IPL_NONE,CPUVAR(ILEVEL) |
jne 3f |
jne 3f |
INTRFASTEXIT |
INTRFASTEXIT |
3: |
3: sti |
STI(si) |
|
movabsq $4f, %rdi |
movabsq $4f, %rdi |
xorq %rax,%rax |
xorq %rax,%rax |
call _C_LABEL(printf) |
call _C_LABEL(printf) |
movl $IPL_NONE,%edi |
#ifdef DDB |
call _C_LABEL(spllower) |
int $3 |
jmp .Losyscall_checkast |
#endif /* DDB */ |
|
movl $IPL_NONE,CPUVAR(ILEVEL) |
|
jmp 2b |
4: .asciz "WARNING: SPL NOT LOWERED ON SYSCALL EXIT\n" |
4: .asciz "WARNING: SPL NOT LOWERED ON SYSCALL EXIT\n" |
#endif /* DIAGNOSTIC */ |
#endif /* DIAGNOSTIC */ |
9: STI(si) |
|
call _C_LABEL(do_pmap_load) |
|
jmp .Losyscall_checkast /* re-check ASTs */ |
|
|
|
/* |
|
* void sse2_zero_page(void *pg) |
|
* |
|
* Zero a page without polluting the cache. |
|
*/ |
|
ENTRY(sse2_zero_page) |
|
movl $PAGE_SIZE, %ecx |
|
xorq %rax, %rax |
|
.align 16 |
|
1: |
|
movnti %rax, 0(%rdi) |
|
movnti %rax, 8(%rdi) |
|
movnti %rax, 16(%rdi) |
|
movnti %rax, 24(%rdi) |
|
movnti %rax, 32(%rdi) |
|
movnti %rax, 40(%rdi) |
|
movnti %rax, 48(%rdi) |
|
movnti %rax, 56(%rdi) |
|
subl $64, %ecx |
|
leaq 64(%rdi), %rdi |
|
jnz 1b |
|
sfence |
|
ret |
|
|
|
/* |
|
* void sse2_copy_page(void *src, void *dst) |
|
* |
|
* Copy a page without polluting the cache. |
|
*/ |
|
ENTRY(sse2_copy_page) |
|
movl $PAGE_SIZE, %ecx |
|
.align 16 |
|
1: |
|
movq 0(%rdi), %rax |
|
movq 8(%rdi), %rdx |
|
movq 16(%rdi), %r8 |
|
movq 24(%rdi), %r9 |
|
movnti %rax, 0(%rsi) |
|
movnti %rdx, 8(%rsi) |
|
movnti %r8, 16(%rsi) |
|
movnti %r9, 24(%rsi) |
|
subl $32, %ecx |
|
leaq 32(%rdi), %rdi |
|
leaq 32(%rsi), %rsi |
|
jnz 1b |
|
sfence |
|
ret |
|