Annotation of src/sys/arch/amd64/amd64/locore.S, Revision 1.150
1.150 ! maxv 1: /* $NetBSD: locore.S,v 1.149 2018/01/22 08:14:09 maxv Exp $ */
1.1 fvdl 2:
3: /*
4: * Copyright-o-rama!
5: */
6:
7: /*
1.84 maxv 8: * Copyright (c) 1998, 2000, 2007, 2008, 2016 The NetBSD Foundation, Inc.
9: * All rights reserved.
10: *
11: * This code is derived from software contributed to The NetBSD Foundation
1.100 maxv 12: * by Charles M. Hannum and by Maxime Villard.
1.84 maxv 13: *
14: * Redistribution and use in source and binary forms, with or without
15: * modification, are permitted provided that the following conditions
16: * are met:
17: * 1. Redistributions of source code must retain the above copyright
18: * notice, this list of conditions and the following disclaimer.
19: * 2. Redistributions in binary form must reproduce the above copyright
20: * notice, this list of conditions and the following disclaimer in the
21: * documentation and/or other materials provided with the distribution.
22: *
23: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
24: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
25: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
26: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
27: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33: * POSSIBILITY OF SUCH DAMAGE.
34: */
35:
36: /*
1.32 bouyer 37: * Copyright (c) 2007 Manuel Bouyer.
38: *
39: * Redistribution and use in source and binary forms, with or without
40: * modification, are permitted provided that the following conditions
41: * are met:
42: * 1. Redistributions of source code must retain the above copyright
43: * notice, this list of conditions and the following disclaimer.
44: * 2. Redistributions in binary form must reproduce the above copyright
45: * notice, this list of conditions and the following disclaimer in the
46: * documentation and/or other materials provided with the distribution.
47: *
48: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
49: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
50: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
51: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
52: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
53: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
54: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
55: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
56: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
57: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
58: *
59: */
60:
61: /*
62: * Copyright (c) 2006 Mathieu Ropert <mro@adviseo.fr>
63: *
64: * Permission to use, copy, modify, and distribute this software for any
65: * purpose with or without fee is hereby granted, provided that the above
66: * copyright notice and this permission notice appear in all copies.
67: *
68: * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
69: * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
70: * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
71: * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
72: * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
73: * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
74: * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
75: */
76:
77: /*
1.1 fvdl 78: * Copyright (c) 2001 Wasabi Systems, Inc.
79: * All rights reserved.
80: *
81: * Written by Frank van der Linden for Wasabi Systems, Inc.
82: *
83: * Redistribution and use in source and binary forms, with or without
84: * modification, are permitted provided that the following conditions
85: * are met:
86: * 1. Redistributions of source code must retain the above copyright
87: * notice, this list of conditions and the following disclaimer.
88: * 2. Redistributions in binary form must reproduce the above copyright
89: * notice, this list of conditions and the following disclaimer in the
90: * documentation and/or other materials provided with the distribution.
91: * 3. All advertising materials mentioning features or use of this software
92: * must display the following acknowledgement:
93: * This product includes software developed for the NetBSD Project by
94: * Wasabi Systems, Inc.
95: * 4. The name of Wasabi Systems, Inc. may not be used to endorse
96: * or promote products derived from this software without specific prior
97: * written permission.
98: *
99: * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
100: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
101: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
102: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
103: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
104: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
105: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
106: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
107: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
108: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
109: * POSSIBILITY OF SUCH DAMAGE.
110: */
111:
112: /*-
113: * Copyright (c) 1990 The Regents of the University of California.
114: * All rights reserved.
115: *
116: * This code is derived from software contributed to Berkeley by
117: * William Jolitz.
118: *
119: * Redistribution and use in source and binary forms, with or without
120: * modification, are permitted provided that the following conditions
121: * are met:
122: * 1. Redistributions of source code must retain the above copyright
123: * notice, this list of conditions and the following disclaimer.
124: * 2. Redistributions in binary form must reproduce the above copyright
125: * notice, this list of conditions and the following disclaimer in the
126: * documentation and/or other materials provided with the distribution.
1.5 agc 127: * 3. Neither the name of the University nor the names of its contributors
1.1 fvdl 128: * may be used to endorse or promote products derived from this software
129: * without specific prior written permission.
130: *
131: * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
132: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
133: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
134: * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
135: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
136: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
137: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
138: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
139: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
140: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
141: * SUCH DAMAGE.
142: *
143: * @(#)locore.s 7.3 (Berkeley) 5/13/91
144: */
145:
1.84 maxv 146: /* Override user-land alignment before including asm.h */
1.34 dsl 147: #define ALIGN_DATA .align 8
148: #define ALIGN_TEXT .align 16,0x90
149: #define _ALIGN_TEXT ALIGN_TEXT
150:
151: #include <machine/asm.h>
152:
1.78 uebayasi 153: #include "opt_copy_symtab.h"
1.1 fvdl 154: #include "opt_ddb.h"
155: #include "opt_ddbparam.h"
1.51 apb 156: #include "opt_modular.h"
1.1 fvdl 157: #include "opt_realmem.h"
158:
1.12 drochner 159: #include "opt_compat_netbsd.h"
160: #include "opt_compat_netbsd32.h"
1.32 bouyer 161: #include "opt_xen.h"
1.145 maxv 162: #include "opt_svs.h"
1.12 drochner 163:
1.1 fvdl 164: #include "assym.h"
165: #include "lapic.h"
166: #include "ioapic.h"
1.2 fvdl 167: #include "ksyms.h"
1.1 fvdl 168:
169: #include <sys/errno.h>
170: #include <sys/syscall.h>
171:
172: #include <machine/pte.h>
173: #include <machine/segments.h>
174: #include <machine/specialreg.h>
175: #include <machine/trap.h>
176: #include <machine/bootinfo.h>
177: #include <machine/frameasm.h>
1.44 ad 178: #include <machine/cputypes.h>
1.1 fvdl 179:
180: #if NLAPIC > 0
181: #include <machine/i82489reg.h>
182: #endif
183:
184: /* Get definitions for IOM_BEGIN, IOM_END, and IOM_SIZE */
185: #include <dev/isa/isareg.h>
186:
1.71 uebayasi 187: #define _RELOC(x) ((x) - KERNBASE)
188: #define RELOC(x) _RELOC(_C_LABEL(x))
189:
1.86 maxv 190: /* 32bit version of PG_NX */
191: #define PG_NX32 0x80000000
192:
1.83 maxv 193: #if L2_SLOT_KERNBASE > 0
194: #define TABLE_L2_ENTRIES (2 * (NKL2_KIMG_ENTRIES + 1))
195: #else
196: #define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1)
197: #endif
198:
199: #if L3_SLOT_KERNBASE > 0
200: #define TABLE_L3_ENTRIES (2 * NKL3_KIMG_ENTRIES)
201: #else
202: #define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES
203: #endif
204:
205: #define PROC0_PML4_OFF 0
1.97 maxv 206: #define PROC0_STK_OFF (PROC0_PML4_OFF + 1 * PAGE_SIZE)
207: #define PROC0_PTP3_OFF (PROC0_STK_OFF + UPAGES * PAGE_SIZE)
1.83 maxv 208: #define PROC0_PTP2_OFF (PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * PAGE_SIZE)
209: #define PROC0_PTP1_OFF (PROC0_PTP2_OFF + TABLE_L3_ENTRIES * PAGE_SIZE)
210: #define TABLESIZE \
1.97 maxv 211: ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \
1.83 maxv 212: * PAGE_SIZE)
213:
1.121 maxv 214: /* Amount of VA used to map the kernel, the syms and the preloaded modules */
215: #define BOOTMAP_VA_SIZE \
216: (NKL2_KIMG_ENTRIES * (1 << L2_SHIFT) - TABLESIZE - IOM_SIZE)
217:
1.83 maxv 218: /*
219: * fillkpt - Fill in a kernel page table
220: * eax = pte (page frame | control | status)
221: * ebx = page table address
222: * ecx = number of pages to map
223: *
224: * Each entry is 8 (PDE_SIZE) bytes long: we must set the 4 upper bytes to 0.
225: */
226: #define fillkpt \
1.94 maxv 227: cmpl $0,%ecx ; /* zero-sized? */ \
228: je 2f ; \
1.91 maxv 229: 1: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \
230: movl %eax,(%ebx) ; /* store phys addr */ \
231: addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
232: addl $PAGE_SIZE,%eax ; /* next phys page */ \
1.94 maxv 233: loop 1b ; \
234: 2: ;
1.83 maxv 235:
1.84 maxv 236: /*
1.89 maxv 237: * fillkpt_nox - Same as fillkpt, but sets the NX/XD bit.
238: */
239: #define fillkpt_nox \
1.94 maxv 240: cmpl $0,%ecx ; /* zero-sized? */ \
241: je 2f ; \
1.91 maxv 242: pushl %ebp ; \
243: movl RELOC(nox_flag),%ebp ; \
244: 1: movl %ebp,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: NX */ \
245: movl %eax,(%ebx) ; /* store phys addr */ \
1.89 maxv 246: addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
247: addl $PAGE_SIZE,%eax ; /* next phys page */ \
1.91 maxv 248: loop 1b ; \
1.94 maxv 249: popl %ebp ; \
250: 2: ;
1.89 maxv 251:
252: /*
1.96 maxv 253: * fillkpt_blank - Fill in a kernel page table with blank entries
254: * ebx = page table address
255: * ecx = number of pages to map
256: */
257: #define fillkpt_blank \
258: cmpl $0,%ecx ; /* zero-sized? */ \
259: je 2f ; \
260: 1: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \
261: movl $0,(%ebx) ; /* lower 32 bits: 0 */ \
262: addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
263: loop 1b ; \
264: 2: ;
265:
266: /*
1.84 maxv 267: * killkpt - Destroy a kernel page table (long mode)
268: * rbx = page table address
269: * rcx = number of pages to destroy
270: */
271: #define killkpt \
272: 1: movq $0,(%rbx) ; \
273: addq $PDE_SIZE,%rbx ; \
274: loop 1b ;
275:
1.83 maxv 276:
1.32 bouyer 277: #ifdef XEN
1.99 bouyer 278: #define __ASSEMBLY__
279: #include <xen/xen-public/elfnote.h>
280: #include <xen/xen-public/xen.h>
281: #define ELFNOTE(name, type, desctype, descdata...) \
282: .pushsection .note.name ; \
283: .align 4 ; \
284: .long 2f - 1f /* namesz */ ; \
285: .long 4f - 3f /* descsz */ ; \
286: .long type ; \
287: 1:.asciz #name ; \
288: 2:.align 4 ; \
289: 3:desctype descdata ; \
290: 4:.align 4 ; \
291: .popsection
292:
1.32 bouyer 293: /*
1.73 uebayasi 294: * Xen guest identifier and loader selection
1.32 bouyer 295: */
296: .section __xen_guest
1.99 bouyer 297: ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS, .asciz, "NetBSD")
298: ELFNOTE(Xen, XEN_ELFNOTE_GUEST_VERSION, .asciz, "4.99")
299: ELFNOTE(Xen, XEN_ELFNOTE_XEN_VERSION, .asciz, "xen-3.0")
300: ELFNOTE(Xen, XEN_ELFNOTE_VIRT_BASE, .quad, KERNBASE)
301: ELFNOTE(Xen, XEN_ELFNOTE_PADDR_OFFSET, .quad, KERNBASE)
302: ELFNOTE(Xen, XEN_ELFNOTE_ENTRY, .quad, start)
303: ELFNOTE(Xen, XEN_ELFNOTE_HYPERCALL_PAGE, .quad, hypercall_page)
304: ELFNOTE(Xen, XEN_ELFNOTE_HV_START_LOW, .quad, HYPERVISOR_VIRT_START)
305: ELFNOTE(Xen, XEN_ELFNOTE_FEATURES, .asciz, "")
306: ELFNOTE(Xen, XEN_ELFNOTE_PAE_MODE, .asciz, "yes")
307: ELFNOTE(Xen, XEN_ELFNOTE_L1_MFN_VALID, .long, PG_V, PG_V)\
308: ELFNOTE(Xen, XEN_ELFNOTE_LOADER, .asciz, "generic")
309: ELFNOTE(Xen, XEN_ELFNOTE_SUSPEND_CANCEL, .long, 0)
1.49 ad 310: #if NKSYMS > 0 || defined(DDB) || defined(MODULAR)
1.99 bouyer 311: ELFNOTE(Xen, XEN_ELFNOTE_BSD_SYMTAB, .asciz, "yes")
1.32 bouyer 312: #endif
313: #endif /* XEN */
1.1 fvdl 314:
315: /*
316: * Initialization
317: */
318: .data
319:
1.98 maxv 320: .globl _C_LABEL(tablesize)
1.86 maxv 321: .globl _C_LABEL(nox_flag)
1.100 maxv 322: .globl _C_LABEL(cputype)
1.57 jym 323: .globl _C_LABEL(cpuid_level)
1.71 uebayasi 324: .globl _C_LABEL(esym)
325: .globl _C_LABEL(eblob)
1.100 maxv 326: .globl _C_LABEL(atdevbase)
327: .globl _C_LABEL(PDPpaddr)
1.71 uebayasi 328: .globl _C_LABEL(boothowto)
329: .globl _C_LABEL(bootinfo)
330: .globl _C_LABEL(biosbasemem)
331: .globl _C_LABEL(biosextmem)
1.130 maxv 332: .globl _C_LABEL(lwp0uarea)
1.128 maxv 333: .globl do_sysret
1.138 maxv 334: .globl do_mov_es
335: .globl do_mov_ds
336: .globl do_mov_fs
337: .globl do_mov_gs
1.129 maxv 338: .globl do_iret
1.43 ad 339:
1.98 maxv 340: .type _C_LABEL(tablesize), @object
341: _C_LABEL(tablesize): .long TABLESIZE
342: END(tablesize)
1.86 maxv 343: .type _C_LABEL(nox_flag), @object
344: LABEL(nox_flag) .long 0 /* 32bit NOX flag, set if supported */
345: END(nox_flag)
1.71 uebayasi 346: .type _C_LABEL(cputype), @object
1.84 maxv 347: LABEL(cputype) .long 0 /* are we 80486, Pentium, or.. */
1.71 uebayasi 348: END(cputype)
349: .type _C_LABEL(cpuid_level), @object
1.84 maxv 350: LABEL(cpuid_level) .long -1 /* max. level accepted by cpuid instr */
1.71 uebayasi 351: END(cpuid_level)
352: .type _C_LABEL(esym), @object
1.84 maxv 353: LABEL(esym) .quad 0 /* ptr to end of syms */
1.71 uebayasi 354: END(esym)
355: .type _C_LABEL(eblob), @object
1.84 maxv 356: LABEL(eblob) .quad 0 /* ptr to end of modules */
1.71 uebayasi 357: END(eblob)
358: .type _C_LABEL(atdevbase), @object
1.84 maxv 359: LABEL(atdevbase) .quad 0 /* location of start of iomem in virt */
1.71 uebayasi 360: END(atdevbase)
361: .type _C_LABEL(PDPpaddr), @object
1.84 maxv 362: LABEL(PDPpaddr) .quad 0 /* paddr of PTD, for libkvm */
1.71 uebayasi 363: END(PDPpaddr)
364: .type _C_LABEL(biosbasemem), @object
1.1 fvdl 365: #ifndef REALBASEMEM
1.84 maxv 366: LABEL(biosbasemem) .long 0 /* base memory reported by BIOS */
1.1 fvdl 367: #else
1.71 uebayasi 368: LABEL(biosbasemem) .long REALBASEMEM
1.1 fvdl 369: #endif
1.71 uebayasi 370: END(biosbasemem)
371: .type _C_LABEL(biosextmem), @object
1.1 fvdl 372: #ifndef REALEXTMEM
1.84 maxv 373: LABEL(biosextmem) .long 0 /* extended memory reported by BIOS */
1.1 fvdl 374: #else
1.71 uebayasi 375: LABEL(biosextmem) .long REALEXTMEM
1.1 fvdl 376: #endif
1.71 uebayasi 377: END(biosextmem)
1.130 maxv 378: .type _C_LABEL(lwp0uarea), @object
379: LABEL(lwp0uarea) .quad 0
380: END(lwp0uarea)
1.1 fvdl 381:
1.32 bouyer 382: #ifndef XEN
1.56 jym 383: .globl gdt64_lo
384: .globl gdt64_hi
1.1 fvdl 385:
1.56 jym 386: #define GDT64_LIMIT gdt64_end-gdt64_start-1
387: /* Temporary gdt64, with base address in low memory */
1.71 uebayasi 388: .type _C_LABEL(gdt64_lo), @object
389: LABEL(gdt64_lo)
1.56 jym 390: .word GDT64_LIMIT
1.1 fvdl 391: .quad _RELOC(gdt64_start)
1.71 uebayasi 392: END(gdt64_lo)
1.1 fvdl 393: .align 64
394:
1.56 jym 395: /* Temporary gdt64, with base address in high memory */
1.71 uebayasi 396: .type _C_LABEL(gdt64_hi), @object
397: LABEL(gdt64_hi)
1.56 jym 398: .word GDT64_LIMIT
399: .quad gdt64_start
1.71 uebayasi 400: END(gdt64_hi)
1.56 jym 401: .align 64
402: #undef GDT64_LIMIT
403:
1.71 uebayasi 404: .type _C_LABEL(gdt64_start), @object
405: _C_LABEL(gdt64_start):
1.1 fvdl 406: .quad 0x0000000000000000 /* always empty */
407: .quad 0x00af9a000000ffff /* kernel CS */
408: .quad 0x00cf92000000ffff /* kernel DS */
1.71 uebayasi 409: END(gdt64_start)
1.1 fvdl 410: gdt64_end:
411:
1.71 uebayasi 412: .type _C_LABEL(farjmp64), @object
413: _C_LABEL(farjmp64):
1.57 jym 414: .long _RELOC(longmode)
1.1 fvdl 415: .word GSEL(GCODE_SEL, SEL_KPL)
1.71 uebayasi 416: END(farjmp64)
1.84 maxv 417:
1.32 bouyer 418: #endif /* !XEN */
1.71 uebayasi 419:
1.84 maxv 420: /* Space for the temporary stack */
1.71 uebayasi 421: .size tmpstk, tmpstk - .
1.84 maxv 422: .space 512
1.1 fvdl 423: tmpstk:
424:
425: /*
426: * Some hackage to deal with 64bit symbols in 32 bit mode.
1.79 maxv 427: * This may not be needed if things are cleaned up a little.
1.1 fvdl 428: */
429:
430: .text
431: .globl _C_LABEL(kernel_text)
432: .set _C_LABEL(kernel_text),KERNTEXTOFF
433:
1.71 uebayasi 434: ENTRY(start)
1.32 bouyer 435: #ifndef XEN
1.71 uebayasi 436: .code32
1.81 maxv 437:
438: /* Warm boot */
439: movw $0x1234,0x472
440:
1.1 fvdl 441: /*
1.84 maxv 442: * Load parameters from the stack (32 bits):
1.91 maxv 443: * boothowto, [bootdev], bootinfo, esym, biosextmem, biosbasemem
1.81 maxv 444: * We are not interested in 'bootdev'.
1.1 fvdl 445: */
1.81 maxv 446:
447: /* Load 'boothowto' */
1.1 fvdl 448: movl 4(%esp),%eax
449: movl %eax,RELOC(boothowto)
1.81 maxv 450:
451: /* Load 'bootinfo' */
1.1 fvdl 452: movl 12(%esp),%eax
1.81 maxv 453: testl %eax,%eax /* bootinfo = NULL? */
1.142 maxv 454: jz .Lbootinfo_finished
1.81 maxv 455:
1.121 maxv 456: movl (%eax),%ebx /* bootinfo::bi_nentries */
1.1 fvdl 457: movl $RELOC(bootinfo),%ebp
1.81 maxv 458: movl %ebp,%edx
1.1 fvdl 459: addl $BOOTINFO_MAXSIZE,%ebp
1.81 maxv 460: movl %ebx,(%edx)
461: addl $4,%edx
462:
1.142 maxv 463: .Lbootinfo_entryloop:
1.81 maxv 464: testl %ebx,%ebx /* no remaining entries? */
1.142 maxv 465: jz .Lbootinfo_finished
1.81 maxv 466:
467: addl $4,%eax
468: movl (%eax),%ecx /* address of entry */
1.1 fvdl 469: pushl %edi
470: pushl %esi
471: pushl %eax
472:
1.84 maxv 473: movl (%ecx),%eax /* btinfo_common::len (size of entry) */
1.1 fvdl 474: movl %edx,%edi
1.121 maxv 475: addl %eax,%edx /* update dest pointer */
1.81 maxv 476: cmpl %ebp,%edx /* beyond bootinfo+BOOTINFO_MAXSIZE? */
1.142 maxv 477: jg .Lbootinfo_overflow
1.81 maxv 478:
1.1 fvdl 479: movl %ecx,%esi
480: movl %eax,%ecx
1.81 maxv 481:
1.43 ad 482: /*
1.121 maxv 483: * If any modules were loaded, record where they end. 'eblob' is used
484: * later to compute the initial bootstrap tables.
1.43 ad 485: */
1.84 maxv 486: cmpl $BTINFO_MODULELIST,4(%esi) /* btinfo_common::type */
1.142 maxv 487: jne .Lbootinfo_copy
1.81 maxv 488:
1.121 maxv 489: /* Skip the modules if we won't have enough VA to map them */
490: movl 12(%esi),%eax /* btinfo_modulelist::endpa */
491: addl $PGOFSET,%eax /* roundup to a page */
492: andl $~PGOFSET,%eax
493: cmpl $BOOTMAP_VA_SIZE,%eax
1.142 maxv 494: jg .Lbootinfo_skip
1.121 maxv 495: movl %eax,RELOC(eblob)
1.81 maxv 496: addl $KERNBASE_LO,RELOC(eblob)
497: adcl $KERNBASE_HI,RELOC(eblob)+4
498:
1.142 maxv 499: .Lbootinfo_copy:
1.1 fvdl 500: rep
1.81 maxv 501: movsb /* copy esi -> edi */
1.142 maxv 502: jmp .Lbootinfo_next
1.121 maxv 503:
1.142 maxv 504: .Lbootinfo_skip:
1.121 maxv 505: subl %ecx,%edx /* revert dest pointer */
506:
1.142 maxv 507: .Lbootinfo_next:
1.1 fvdl 508: popl %eax
509: popl %esi
510: popl %edi
1.81 maxv 511: subl $1,%ebx /* decrement the # of entries */
1.142 maxv 512: jmp .Lbootinfo_entryloop
1.81 maxv 513:
1.142 maxv 514: .Lbootinfo_overflow:
1.81 maxv 515: /*
516: * Cleanup for overflow case. Pop the registers, and correct the number
517: * of entries.
518: */
1.1 fvdl 519: popl %eax
520: popl %esi
521: popl %edi
522: movl $RELOC(bootinfo),%ebp
1.81 maxv 523: movl %ebp,%edx
524: subl %ebx,(%edx) /* correct the number of entries */
1.142 maxv 525: .Lbootinfo_finished:
1.1 fvdl 526:
1.81 maxv 527: /* Load 'esym' */
1.72 uebayasi 528: movl 16(%esp),%eax
1.81 maxv 529: testl %eax,%eax /* esym = NULL? */
1.1 fvdl 530: jz 1f
1.81 maxv 531:
1.1 fvdl 532: addl $KERNBASE_LO,%eax
1.81 maxv 533:
534: 1:
535: movl $RELOC(esym),%ebp
1.1 fvdl 536: movl %eax,(%ebp)
537: movl $KERNBASE_HI,4(%ebp)
538:
1.83 maxv 539: /* Load 'biosextmem' */
1.1 fvdl 540: movl $RELOC(biosextmem),%ebp
541: movl (%ebp),%eax
1.83 maxv 542: testl %eax,%eax /* already set? */
1.142 maxv 543: jnz .Lbiosextmem_finished
1.81 maxv 544:
1.1 fvdl 545: movl 20(%esp),%eax
546: movl %eax,(%ebp)
1.81 maxv 547:
1.142 maxv 548: .Lbiosextmem_finished:
1.83 maxv 549: /* Load 'biosbasemem' */
1.1 fvdl 550: movl $RELOC(biosbasemem),%ebp
551: movl (%ebp),%eax
1.83 maxv 552: testl %eax,%eax /* already set? */
1.142 maxv 553: jnz .Lbiosbasemem_finished
1.83 maxv 554:
1.1 fvdl 555: movl 24(%esp),%eax
556: movl %eax,(%ebp)
557:
1.142 maxv 558: .Lbiosbasemem_finished:
1.83 maxv 559: /*
1.84 maxv 560: * Done with the parameters!
1.83 maxv 561: */
1.84 maxv 562:
563: /* First, reset the PSL. */
1.1 fvdl 564: pushl $PSL_MBO
565: popfl
566:
567: xorl %eax,%eax
568: cpuid
569: movl %eax,RELOC(cpuid_level)
570:
571: /*
572: * Finished with old stack; load new %esp now instead of later so we
573: * can trace this code without having to worry about the trace trap
574: * clobbering the memory test or the zeroing of the bss+bootstrap page
575: * tables.
576: *
577: * The boot program should check:
578: * text+data <= &stack_variable - more_space_for_stack
579: * text+data+bss+pad+space_for_page_tables <= end_of_memory
1.100 maxv 580: *
1.84 maxv 581: * XXX: the gdt is in the carcass of the boot program so clearing
1.1 fvdl 582: * the rest of memory is still not possible.
583: */
584: movl $RELOC(tmpstk),%esp
585:
1.86 maxv 586: /*
587: * Retrieve the NX/XD flag. We use the 32bit version of PG_NX.
588: */
589: movl $0x80000001,%eax
590: cpuid
591: andl $CPUID_NOX,%edx
1.142 maxv 592: jz .Lno_NOX
1.86 maxv 593: movl $PG_NX32,RELOC(nox_flag)
1.142 maxv 594: .Lno_NOX:
1.86 maxv 595:
1.1 fvdl 596: /*
1.84 maxv 597: * There are four levels of pages in amd64: PML4 -> PDP -> PD -> PT. They will
598: * be referred to as: L4 -> L3 -> L2 -> L1.
599: *
600: * Virtual address space of the kernel:
1.97 maxv 601: * +------+--------+------+-----+--------+---------------------+----------
602: * | TEXT | RODATA | DATA | BSS | [SYMS] | [PRELOADED MODULES] | L4 ->
603: * +------+--------+------+-----+--------+---------------------+----------
604: * (1) (2) (3)
605: *
1.101 maxv 606: * --------------+-----+-----+----+-------------+
607: * -> PROC0 STK -> L3 -> L2 -> L1 | ISA I/O MEM |
608: * --------------+-----+-----+----+-------------+
1.97 maxv 609: * (4)
1.84 maxv 610: *
1.97 maxv 611: * PROC0 STK is obviously not linked as a page level. It just happens to be
612: * caught between L4 and L3.
613: *
614: * (PROC0 STK + L4 + L3 + L2 + L1) is later referred to as BOOTSTRAP TABLES.
1.1 fvdl 615: *
1.116 maxv 616: * ISA I/O MEM has no physical page allocated here, just virtual addresses.
617: *
1.88 maxv 618: * Important note: the kernel segments are properly 4k-aligned
619: * (see kern.ldscript), so there's no need to enforce alignment.
1.1 fvdl 620: */
621:
1.84 maxv 622: /* Find end of kernel image; brings us on (1). */
1.115 maxv 623: movl $RELOC(__kernel_end),%edi
1.83 maxv 624:
1.78 uebayasi 625: #if (NKSYMS || defined(DDB) || defined(MODULAR)) && !defined(makeoptions_COPY_SYMTAB)
1.84 maxv 626: /* Save the symbols (if loaded); brinds us on (2). */
1.1 fvdl 627: movl RELOC(esym),%eax
628: testl %eax,%eax
629: jz 1f
630: subl $KERNBASE_LO,%eax /* XXX */
631: movl %eax,%edi
632: 1:
633: #endif
1.84 maxv 634: /* Skip over any modules/blobs; brings us on (3). */
1.43 ad 635: movl RELOC(eblob),%eax
636: testl %eax,%eax
637: jz 1f
638: subl $KERNBASE_LO,%eax /* XXX */
639: movl %eax,%edi
640: 1:
1.81 maxv 641:
1.97 maxv 642: /* We are on (3). Align up for BOOTSTRAP TABLES. */
1.1 fvdl 643: movl %edi,%esi
1.84 maxv 644: addl $PGOFSET,%esi
1.1 fvdl 645: andl $~PGOFSET,%esi
646:
1.93 maxv 647: /* We are on the BOOTSTRAP TABLES. Save L4's physical address. */
648: movl $RELOC(PDPpaddr),%ebp
649: movl %esi,(%ebp)
650: movl $0,4(%ebp)
651:
1.84 maxv 652: /* Now, zero out the BOOTSTRAP TABLES (before filling them in). */
1.91 maxv 653: movl %esi,%edi
1.1 fvdl 654: xorl %eax,%eax
655: cld
656: movl $TABLESIZE,%ecx
657: shrl $2,%ecx
658: rep
1.83 maxv 659: stosl /* copy eax -> edi */
1.1 fvdl 660:
1.73 uebayasi 661: /*
1.84 maxv 662: * Build the page tables and levels. We go from L1 to L4, and link the levels
663: * together. Note: RELOC computes &addr - KERNBASE in 32 bits; the value can't
664: * be > 4G, or we can't deal with it anyway, since we are in 32bit mode.
1.73 uebayasi 665: */
1.84 maxv 666: /*
667: * Build L1.
668: */
1.83 maxv 669: leal (PROC0_PTP1_OFF)(%esi),%ebx
1.81 maxv 670:
1.119 maxv 671: /* Skip the area below the kernel text. */
1.96 maxv 672: movl $(KERNTEXTOFF_LO - KERNBASE_LO),%ecx
673: shrl $PGSHIFT,%ecx
674: fillkpt_blank
1.81 maxv 675:
1.88 maxv 676: /* Map the kernel text RX. */
1.96 maxv 677: movl $(KERNTEXTOFF_LO - KERNBASE_LO),%eax /* start of TEXT */
1.88 maxv 678: movl $RELOC(__rodata_start),%ecx
679: subl %eax,%ecx
680: shrl $PGSHIFT,%ecx
681: orl $(PG_V|PG_KR),%eax
682: fillkpt
1.1 fvdl 683:
1.91 maxv 684: /* Map the kernel rodata R. */
1.88 maxv 685: movl $RELOC(__rodata_start),%eax
686: movl $RELOC(__data_start),%ecx
1.1 fvdl 687: subl %eax,%ecx
688: shrl $PGSHIFT,%ecx
1.73 uebayasi 689: orl $(PG_V|PG_KR),%eax
1.89 maxv 690: fillkpt_nox
1.1 fvdl 691:
1.90 maxv 692: /* Map the kernel data+bss RW. */
693: movl $RELOC(__data_start),%eax
694: movl $RELOC(__kernel_end),%ecx
695: subl %eax,%ecx
696: shrl $PGSHIFT,%ecx
697: orl $(PG_V|PG_KW),%eax
698: fillkpt_nox
699:
1.105 maxv 700: /* Map [SYMS]+[PRELOADED MODULES] RW. */
1.90 maxv 701: movl $RELOC(__kernel_end),%eax
1.97 maxv 702: movl %esi,%ecx /* start of BOOTSTRAP TABLES */
1.95 maxv 703: subl %eax,%ecx
704: shrl $PGSHIFT,%ecx
705: orl $(PG_V|PG_KW),%eax
1.105 maxv 706: fillkpt_nox
1.95 maxv 707:
1.92 maxv 708: /* Map the BOOTSTRAP TABLES RW. */
709: movl %esi,%eax /* start of BOOTSTRAP TABLES */
710: movl $TABLESIZE,%ecx /* length of BOOTSTRAP TABLES */
711: shrl $PGSHIFT,%ecx
712: orl $(PG_V|PG_KW),%eax
713: fillkpt_nox
714:
1.102 maxv 715: /* We are on (4). Map ISA I/O MEM RW. */
1.101 maxv 716: movl $IOM_BEGIN,%eax
717: movl $IOM_SIZE,%ecx /* size of ISA I/O MEM */
718: shrl $PGSHIFT,%ecx
719: orl $(PG_V|PG_KW/*|PG_N*/),%eax
1.102 maxv 720: fillkpt_nox
1.1 fvdl 721:
1.84 maxv 722: /*
723: * Build L2. Linked to L1.
724: */
1.73 uebayasi 725: leal (PROC0_PTP2_OFF)(%esi),%ebx
1.1 fvdl 726: leal (PROC0_PTP1_OFF)(%esi),%eax
1.81 maxv 727: orl $(PG_V|PG_KW),%eax
1.1 fvdl 728: movl $(NKL2_KIMG_ENTRIES+1),%ecx
729: fillkpt
730:
731: #if L2_SLOT_KERNBASE > 0
732: /* If needed, set up level 2 entries for actual kernel mapping */
1.84 maxv 733: leal (PROC0_PTP2_OFF + L2_SLOT_KERNBASE * PDE_SIZE)(%esi),%ebx
1.73 uebayasi 734: leal (PROC0_PTP1_OFF)(%esi),%eax
1.81 maxv 735: orl $(PG_V|PG_KW),%eax
1.73 uebayasi 736: movl $(NKL2_KIMG_ENTRIES+1),%ecx
1.1 fvdl 737: fillkpt
738: #endif
739:
1.84 maxv 740: /*
741: * Build L3. Linked to L2.
742: */
1.73 uebayasi 743: leal (PROC0_PTP3_OFF)(%esi),%ebx
1.1 fvdl 744: leal (PROC0_PTP2_OFF)(%esi),%eax
1.81 maxv 745: orl $(PG_V|PG_KW),%eax
1.1 fvdl 746: movl $NKL3_KIMG_ENTRIES,%ecx
747: fillkpt
748:
749: #if L3_SLOT_KERNBASE > 0
750: /* If needed, set up level 3 entries for actual kernel mapping */
1.84 maxv 751: leal (PROC0_PTP3_OFF + L3_SLOT_KERNBASE * PDE_SIZE)(%esi),%ebx
1.73 uebayasi 752: leal (PROC0_PTP2_OFF)(%esi),%eax
1.83 maxv 753: orl $(PG_V|PG_KW),%eax
1.73 uebayasi 754: movl $NKL3_KIMG_ENTRIES,%ecx
1.1 fvdl 755: fillkpt
756: #endif
757:
1.84 maxv 758: /*
759: * Build L4 for identity mapping. Linked to L3.
760: */
1.73 uebayasi 761: leal (PROC0_PML4_OFF)(%esi),%ebx
1.1 fvdl 762: leal (PROC0_PTP3_OFF)(%esi),%eax
1.81 maxv 763: orl $(PG_V|PG_KW),%eax
1.1 fvdl 764: movl $NKL4_KIMG_ENTRIES,%ecx
765: fillkpt
766:
1.84 maxv 767: /* Set up L4 entries for actual kernel mapping */
768: leal (PROC0_PML4_OFF + L4_SLOT_KERNBASE * PDE_SIZE)(%esi),%ebx
1.1 fvdl 769: leal (PROC0_PTP3_OFF)(%esi),%eax
1.83 maxv 770: orl $(PG_V|PG_KW),%eax
1.1 fvdl 771: movl $NKL4_KIMG_ENTRIES,%ecx
772: fillkpt
773:
1.106 maxv 774: /* Install recursive top level PDE (one entry) */
1.84 maxv 775: leal (PROC0_PML4_OFF + PDIR_SLOT_PTE * PDE_SIZE)(%esi),%ebx
1.73 uebayasi 776: leal (PROC0_PML4_OFF)(%esi),%eax
1.1 fvdl 777: orl $(PG_V|PG_KW),%eax
1.106 maxv 778: movl $1,%ecx
779: fillkpt_nox
1.1 fvdl 780:
781: /*
782: * Startup checklist:
783: * 1. Enable PAE (and SSE while here).
784: */
785: movl %cr4,%eax
786: orl $(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax
787: movl %eax,%cr4
788:
789: /*
1.86 maxv 790: * 2. Set Long Mode Enable in EFER. Also enable the syscall extensions,
791: * and NOX if available.
1.1 fvdl 792: */
1.73 uebayasi 793: movl $MSR_EFER,%ecx
1.1 fvdl 794: rdmsr
795: xorl %eax,%eax /* XXX */
796: orl $(EFER_LME|EFER_SCE),%eax
1.86 maxv 797: movl RELOC(nox_flag),%ebx
798: cmpl $0,%ebx
1.142 maxv 799: je .Lskip_NOX
1.86 maxv 800: orl $(EFER_NXE),%eax
1.142 maxv 801: .Lskip_NOX:
1.1 fvdl 802: wrmsr
803:
804: /*
805: * 3. Load %cr3 with pointer to PML4.
806: */
807: movl %esi,%eax
808: movl %eax,%cr3
809:
810: /*
811: * 4. Enable paging and the rest of it.
812: */
813: movl %cr0,%eax
1.68 jym 814: orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax
1.1 fvdl 815: movl %eax,%cr0
816: jmp compat
817: compat:
818:
819: /*
1.83 maxv 820: * 5. Not quite done yet, we're now in a compatibility segment, in
821: * legacy mode. We must jump to a long mode segment. Need to set up
822: * a temporary GDT with a long mode segment in it to do that.
1.1 fvdl 823: */
1.56 jym 824: movl $RELOC(gdt64_lo),%eax
1.1 fvdl 825: lgdt (%eax)
826: movl $RELOC(farjmp64),%eax
827: ljmp *(%eax)
828:
1.83 maxv 829: .code64
1.1 fvdl 830: longmode:
831: /*
1.83 maxv 832: * 6. Finally, we're in long mode. However, we're still in the identity
833: * mapped area (could not jump out of that earlier because it would
834: * have been a > 32bit jump). We can do that now, so here we go.
1.1 fvdl 835: */
836: movabsq $longmode_hi,%rax
837: jmp *%rax
1.56 jym 838:
1.1 fvdl 839: longmode_hi:
1.56 jym 840:
841: /*
842: * We left the identity mapped area. Base address of
843: * the temporary gdt64 should now be in high memory.
844: */
845: movq $RELOC(gdt64_hi),%rax
846: lgdt (%rax)
847:
1.1 fvdl 848: /*
1.83 maxv 849: * We have arrived. There's no need anymore for the identity mapping in
850: * low memory, remove it.
1.1 fvdl 851: */
852: movq $KERNBASE,%r8
853:
854: #if L2_SLOT_KERNBASE > 0
855: movq $(NKL2_KIMG_ENTRIES+1),%rcx
1.84 maxv 856: leaq (PROC0_PTP2_OFF)(%rsi),%rbx /* old, phys address */
857: addq %r8,%rbx /* new, virt address */
858: killkpt
1.1 fvdl 859: #endif
860:
861: #if L3_SLOT_KERNBASE > 0
862: movq $NKL3_KIMG_ENTRIES,%rcx
1.84 maxv 863: leaq (PROC0_PTP3_OFF)(%rsi),%rbx /* old, phys address */
864: addq %r8,%rbx /* new, virt address */
865: killkpt
1.1 fvdl 866: #endif
867:
868: movq $NKL4_KIMG_ENTRIES,%rcx
1.84 maxv 869: leaq (PROC0_PML4_OFF)(%rsi),%rbx /* old, phys address of PML4 */
870: addq %r8,%rbx /* new, virt address of PML4 */
1.85 maxv 871: killkpt
1.1 fvdl 872:
873: /* Relocate atdevbase. */
874: movq $(TABLESIZE+KERNBASE),%rdx
875: addq %rsi,%rdx
876: movq %rdx,_C_LABEL(atdevbase)(%rip)
877:
878: /* Set up bootstrap stack. */
1.97 maxv 879: leaq (PROC0_STK_OFF)(%rsi),%rax
1.1 fvdl 880: addq %r8,%rax
1.130 maxv 881: movq %rax,_C_LABEL(lwp0uarea)(%rip)
1.1 fvdl 882: leaq (USPACE-FRAMESIZE)(%rax),%rsp
1.84 maxv 883: xorq %rbp,%rbp /* mark end of frames */
1.1 fvdl 884:
885: xorw %ax,%ax
886: movw %ax,%gs
887: movw %ax,%fs
888:
1.116 maxv 889: /* The first physical page available. */
890: leaq (TABLESIZE)(%rsi),%rdi
1.32 bouyer 891:
892: #else /* XEN */
893: /* First, reset the PSL. */
894: pushq $2
895: popfq
896:
897: cld
898:
899: /*
900: * Xen info:
901: * - %rsi -> start_info struct
1.111 maxv 902: * - %rsp -> stack, *theoretically* the last used page by Xen bootstrap
1.32 bouyer 903: */
1.111 maxv 904: movq %rsi,%rbx
1.32 bouyer 905:
1.73 uebayasi 906: /* Clear BSS. */
1.32 bouyer 907: xorq %rax,%rax
908: movq $_C_LABEL(__bss_start),%rdi
909: movq $_C_LABEL(_end),%rcx
910: subq %rdi,%rcx
911: rep
912: stosb
913:
1.111 maxv 914: /* Copy start_info to a safe place. */
1.32 bouyer 915: movq %rbx,%rsi
916: movq $_C_LABEL(start_info_union),%rdi
917: movq $64,%rcx
918: rep
919: movsq
920:
921: /*
922: * Memory layout at start of the day:
923: * - Kernel image
924: * - Page frames list
925: * - start_info struct. we copied it, so it can be recycled.
926: * - xenstore
927: * - console
928: * - Xen bootstrap page tables
929: * - kernel stack. provided by Xen
1.79 maxv 930: * - guaranteed 512kB padding
1.32 bouyer 931: *
932: * As we want to rebuild our page tables and place our stack
933: * in proc0 struct, all data starting from after console can be
934: * discarded after we've done a little setup.
935: */
936:
937: /*
1.111 maxv 938: * We want our own page tables, and will rebuild them. We will reclaim
939: * the Xen space later, INCLUDING the stack. So we need to switch to a
940: * temporary one now.
941: */
942: movq $tmpstk,%rax
943: subq $8,%rax
944: movq %rax,%rsp
1.32 bouyer 945:
1.45 bouyer 946: xorl %eax,%eax
947: cpuid
948: movl %eax,_C_LABEL(cpuid_level)
949:
1.111 maxv 950: movq $cpu_info_primary,%rdi
951: movq %rdi,CPU_INFO_SELF(%rdi) /* ci->ci_self = ci */
952: movq $1,%rsi
1.67 cherry 953: call cpu_init_msrs /* cpu_init_msrs(ci, true); */
954:
1.113 maxv 955: call xen_locore
1.32 bouyer 956:
957: /*
1.113 maxv 958: * The first VA available is returned by xen_locore in %rax. We
1.111 maxv 959: * use it as the UAREA, and set up the stack here.
1.32 bouyer 960: */
1.111 maxv 961: movq %rax,%rsi
1.130 maxv 962: movq %rsi,_C_LABEL(lwp0uarea)(%rip)
1.32 bouyer 963: leaq (USPACE-FRAMESIZE)(%rsi),%rsp
964: xorq %rbp,%rbp
965:
1.111 maxv 966: /* Clear segment registers. */
1.63 jym 967: xorw %ax,%ax
1.32 bouyer 968: movw %ax,%gs
969: movw %ax,%fs
1.84 maxv 970:
1.117 maxv 971: /* Set first_avail after the DUMMY PAGE (see xen_locore). */
1.32 bouyer 972: movq %rsi,%rdi
1.117 maxv 973: addq $(USPACE+PAGE_SIZE),%rdi
1.84 maxv 974: subq $KERNBASE,%rdi /* init_x86_64 wants a physical address */
1.107 maxv 975: #endif /* XEN */
1.32 bouyer 976:
1.131 maxv 977: pushq %rdi
978: call _C_LABEL(init_bootspace)
979: popq %rdi
1.1 fvdl 980: call _C_LABEL(init_x86_64)
981: call _C_LABEL(main)
1.71 uebayasi 982: END(start)
1.1 fvdl 983:
1.73 uebayasi 984: #if defined(XEN)
1.32 bouyer 985: /* space for the hypercall call page */
986: #define HYPERCALL_PAGE_OFFSET 0x1000
987: .org HYPERCALL_PAGE_OFFSET
988: ENTRY(hypercall_page)
989: .skip 0x1000
1.71 uebayasi 990: END(hypercall_page)
1.32 bouyer 991: #endif /* XEN */
1.1 fvdl 992:
993: /*
1.22 yamt 994: * int setjmp(label_t *)
995: *
996: * Used primarily by DDB.
997: */
1.1 fvdl 998: ENTRY(setjmp)
999: /*
1000: * Only save registers that must be preserved across function
1001: * calls according to the ABI (%rbx, %rsp, %rbp, %r12-%r15)
1002: * and %rip.
1003: */
1004: movq %rdi,%rax
1005: movq %rbx,(%rax)
1006: movq %rsp,8(%rax)
1007: movq %rbp,16(%rax)
1008: movq %r12,24(%rax)
1009: movq %r13,32(%rax)
1010: movq %r14,40(%rax)
1011: movq %r15,48(%rax)
1012: movq (%rsp),%rdx
1013: movq %rdx,56(%rax)
1014: xorl %eax,%eax
1015: ret
1.71 uebayasi 1016: END(setjmp)
1.1 fvdl 1017:
1.22 yamt 1018: /*
1019: * int longjmp(label_t *)
1020: *
1021: * Used primarily by DDB.
1022: */
1.1 fvdl 1023: ENTRY(longjmp)
1024: movq %rdi,%rax
1025: movq (%rax),%rbx
1026: movq 8(%rax),%rsp
1027: movq 16(%rax),%rbp
1028: movq 24(%rax),%r12
1029: movq 32(%rax),%r13
1030: movq 40(%rax),%r14
1031: movq 48(%rax),%r15
1032: movq 56(%rax),%rdx
1033: movq %rdx,(%rsp)
1.22 yamt 1034: movl $1,%eax
1.1 fvdl 1035: ret
1.71 uebayasi 1036: END(longjmp)
1.1 fvdl 1037:
1.73 uebayasi 1038: /*
1039: * void dumpsys(void)
1040: *
1041: * Mimic cpu_switchto() for postmortem debugging.
1042: */
1.25 yamt 1043: ENTRY(dumpsys)
1.84 maxv 1044: /* Build a fake switch frame. */
1.25 yamt 1045: pushq %rbx
1046: pushq %r12
1047: pushq %r13
1048: pushq %r14
1049: pushq %r15
1.84 maxv 1050:
1051: /* Save a context. */
1.25 yamt 1052: movq $dumppcb, %rax
1053: movq %rsp, PCB_RSP(%rax)
1054: movq %rbp, PCB_RBP(%rax)
1055:
1056: call _C_LABEL(dodumpsys)
1057:
1.84 maxv 1058: addq $(5*8), %rsp /* sizeof(switchframe) - sizeof(%rip) */
1.25 yamt 1059: ret
1.71 uebayasi 1060: END(dumpsys)
1.25 yamt 1061:
1.1 fvdl 1062: /*
1.58 chs 1063: * struct lwp *cpu_switchto(struct lwp *oldlwp, struct lwp *newlwp,
1.103 maxv 1064: * bool returning)
1.22 yamt 1065: *
1066: * 1. if (oldlwp != NULL), save its context.
1067: * 2. then, restore context of newlwp.
1068: *
1069: * Note that the stack frame layout is known to "struct switchframe" in
1070: * <machine/frame.h> and to the code in cpu_lwp_fork() which initializes
1071: * it for a new lwp.
1.1 fvdl 1072: */
1.22 yamt 1073: ENTRY(cpu_switchto)
1.1 fvdl 1074: pushq %rbx
1075: pushq %r12
1076: pushq %r13
1077: pushq %r14
1078: pushq %r15
1079:
1.84 maxv 1080: movq %rdi,%r13 /* oldlwp */
1081: movq %rsi,%r12 /* newlwp */
1.1 fvdl 1082:
1.103 maxv 1083: testq %r13,%r13 /* oldlwp = NULL ? */
1.142 maxv 1084: jz .Lskip_save
1.1 fvdl 1085:
1.22 yamt 1086: /* Save old context. */
1.55 rmind 1087: movq L_PCB(%r13),%rax
1.22 yamt 1088: movq %rsp,PCB_RSP(%rax)
1089: movq %rbp,PCB_RBP(%rax)
1.142 maxv 1090: .Lskip_save:
1.22 yamt 1091:
1.145 maxv 1092: #ifdef SVS
1093: pushq %rdx
1094: callq _C_LABEL(svs_lwp_switch)
1095: popq %rdx
1096: #endif
1097:
1.22 yamt 1098: /* Switch to newlwp's stack. */
1.103 maxv 1099: movq L_PCB(%r12),%r14
1.22 yamt 1100: movq PCB_RSP(%r14),%rsp
1101: movq PCB_RBP(%r14),%rbp
1.1 fvdl 1102:
1.39 ad 1103: /*
1104: * Set curlwp. This must be globally visible in order to permit
1105: * non-interlocked mutex release.
1106: */
1107: movq %r12,%rcx
1108: xchgq %rcx,CPUVAR(CURLWP)
1.35 ad 1109:
1110: /* Skip the rest if returning to a pinned LWP. */
1.103 maxv 1111: testb %dl,%dl /* returning = true ? */
1.142 maxv 1112: jnz .Lswitch_return
1.35 ad 1113:
1.38 yamt 1114: /* Switch ring0 stack */
1.147 maxv 1115: #ifdef SVS
1116: movq CPUVAR(RSP0),%rax
1117: movq CPUVAR(TSS),%rdi
1118: movq %rax,TSS_RSP0(%rdi)
1119: #elif !defined(XEN)
1.38 yamt 1120: movq PCB_RSP0(%r14),%rax
1.144 maxv 1121: movq CPUVAR(TSS),%rdi
1122: movq %rax,TSS_RSP0(%rdi)
1.38 yamt 1123: #else
1.103 maxv 1124: movq %r14,%rdi
1.32 bouyer 1125: callq _C_LABEL(x86_64_switch_context);
1126: #endif
1.1 fvdl 1127:
1.22 yamt 1128: /* Don't bother with the rest if switching to a system process. */
1129: testl $LW_SYSTEM,L_FLAG(%r12)
1.142 maxv 1130: jnz .Lswitch_return
1.1 fvdl 1131:
1.22 yamt 1132: /* Is this process using RAS (restartable atomic sequences)? */
1133: movq L_PROC(%r12),%rdi
1134: cmpq $0,P_RASLIST(%rdi)
1.142 maxv 1135: je .Lno_RAS
1.104 maxv 1136:
1137: /* Handle restartable atomic sequences (RAS). */
1138: movq L_MD_REGS(%r12),%rbx
1139: movq TF_RIP(%rbx),%rsi
1140: call _C_LABEL(ras_lookup)
1141: cmpq $-1,%rax
1.142 maxv 1142: je .Lno_RAS
1.104 maxv 1143: movq %rax,TF_RIP(%rbx)
1.142 maxv 1144: .Lno_RAS:
1.1 fvdl 1145:
1146: /*
1.76 rmind 1147: * Restore cr0 including FPU state (may have CR0_TS set). Note that
1148: * IPL_SCHED prevents from FPU interrupt altering the LWP's saved cr0.
1.1 fvdl 1149: */
1.32 bouyer 1150: #ifndef XEN
1.75 christos 1151: movl $IPL_HIGH,CPUVAR(ILEVEL)
1152: movl PCB_CR0(%r14),%ecx /* has CR0_TS clear */
1.22 yamt 1153: movq %cr0,%rdx
1.1 fvdl 1154:
1.75 christos 1155: /*
1156: * If our floating point registers are on a different CPU,
1157: * set CR0_TS so we'll trap rather than reuse bogus state.
1158: */
1159: cmpq CPUVAR(FPCURLWP),%r12
1.142 maxv 1160: je .Lskip_TS
1.75 christos 1161: orq $CR0_TS,%rcx
1.142 maxv 1162: .Lskip_TS:
1.75 christos 1163:
1.22 yamt 1164: /* Reloading CR0 is very expensive - avoid if possible. */
1.103 maxv 1165: cmpq %rdx,%rcx
1.142 maxv 1166: je .Lskip_CR0
1.1 fvdl 1167: movq %rcx,%cr0
1.142 maxv 1168: .Lskip_CR0:
1.58 chs 1169:
1.103 maxv 1170: /* The 32bit LWPs are handled differently. */
1171: testl $PCB_COMPAT32,PCB_FLAGS(%r14)
1.142 maxv 1172: jnz .Llwp_32bit
1.58 chs 1173:
1.142 maxv 1174: .Llwp_64bit:
1.134 maxv 1175: /* Set default 64bit values in %ds, %es, %fs and %gs. */
1176: movq $GSEL(GUDATA_SEL, SEL_UPL),%rax
1177: movw %ax,%ds
1178: movw %ax,%es
1.103 maxv 1179: xorq %rax,%rax
1180: movw %ax,%fs
1.58 chs 1181: CLI(cx)
1.70 dsl 1182: SWAPGS
1.103 maxv 1183: movw %ax,%gs
1.70 dsl 1184: SWAPGS
1.58 chs 1185: STI(cx)
1186:
1.107 maxv 1187: /* Zero out GDT descriptors. */
1.58 chs 1188: movq CPUVAR(GDT),%rcx
1.103 maxv 1189: movq %rax,(GUFS_SEL*8)(%rcx)
1190: movq %rax,(GUGS_SEL*8)(%rcx)
1.58 chs 1191:
1192: /* Reload 64-bit %fs/%gs MSRs. */
1.103 maxv 1193: movl $MSR_FSBASE,%ecx
1194: movl PCB_FS(%r14),%eax
1195: movl 4+PCB_FS(%r14),%edx
1.58 chs 1196: wrmsr
1.103 maxv 1197: movl $MSR_KERNELGSBASE,%ecx
1198: movl PCB_GS(%r14),%eax
1199: movl 4+PCB_GS(%r14),%edx
1.58 chs 1200: wrmsr
1.107 maxv 1201:
1.142 maxv 1202: jmp .Lswitch_return
1.58 chs 1203:
1.142 maxv 1204: .Llwp_32bit:
1.58 chs 1205: /* Reload %fs/%gs GDT descriptors. */
1206: movq CPUVAR(GDT),%rcx
1.103 maxv 1207: movq PCB_FS(%r14),%rax
1208: movq %rax,(GUFS_SEL*8)(%rcx)
1209: movq PCB_GS(%r14),%rax
1210: movq %rax,(GUGS_SEL*8)(%rcx)
1.58 chs 1211:
1.143 maxv 1212: /* Set default 32bit values in %ds, %es, %fs and %gs. */
1.103 maxv 1213: movq L_MD_REGS(%r12),%rbx
1.134 maxv 1214: movq $GSEL(GUDATA32_SEL, SEL_UPL),%rax
1215: movw %ax,%ds
1216: movw %ax,%es
1.135 maxv 1217: movw %ax,%fs
1.58 chs 1218: CLI(ax)
1.70 dsl 1219: SWAPGS
1.137 maxv 1220: movw %ax,%gs
1.70 dsl 1221: SWAPGS
1.58 chs 1222: STI(ax)
1223: #else
1224: movq %r12,%rdi
1225: callq _C_LABEL(x86_64_tls_switch)
1.32 bouyer 1226: #endif
1.84 maxv 1227:
1.142 maxv 1228: .Lswitch_return:
1.22 yamt 1229: /* Return to the new LWP, returning 'oldlwp' in %rax. */
1.103 maxv 1230: movq %r13,%rax
1.1 fvdl 1231: popq %r15
1232: popq %r14
1233: popq %r13
1234: popq %r12
1235: popq %rbx
1236: ret
1.71 uebayasi 1237: END(cpu_switchto)
1.1 fvdl 1238:
1239: /*
1.22 yamt 1240: * void savectx(struct pcb *pcb);
1241: *
1.1 fvdl 1242: * Update pcb, saving current processor state.
1243: */
1244: ENTRY(savectx)
1245: /* Save stack pointers. */
1246: movq %rsp,PCB_RSP(%rdi)
1247: movq %rbp,PCB_RBP(%rdi)
1248: ret
1.71 uebayasi 1249: END(savectx)
1.1 fvdl 1250:
1251: /*
1.148 maxv 1252: * Syscall handler.
1.1 fvdl 1253: */
1.148 maxv 1254: NENTRY(handle_syscall)
1.127 maxv 1255: STI(si)
1.1 fvdl 1256:
1257: movq CPUVAR(CURLWP),%r14
1.84 maxv 1258: incq CPUVAR(NSYSCALL) /* count it atomically */
1259: movq %rsp,L_MD_REGS(%r14) /* save pointer to frame */
1.1 fvdl 1260: movq L_PROC(%r14),%r15
1.70 dsl 1261: andl $~MDL_IRET,L_MD_FLAGS(%r14) /* Allow sysret return */
1.37 dsl 1262: movq %rsp,%rdi /* Pass frame as arg0 */
1.1 fvdl 1263: call *P_MD_SYSCALL(%r15)
1.29 yamt 1264: .Lsyscall_checkast:
1.70 dsl 1265: /*
1266: * Disable interrupts to avoid new ASTs (etc) being added and
1267: * to ensure we don't take an interrupt with some of the user
1268: * registers loaded.
1269: */
1270: CLI(si)
1.29 yamt 1271: /* Check for ASTs on exit to user mode. */
1.107 maxv 1272: movl L_MD_ASTPENDING(%r14),%eax
1273: orl CPUVAR(WANT_PMAPLOAD),%eax
1.29 yamt 1274: jnz 9f
1.107 maxv 1275:
1.1 fvdl 1276: #ifdef DIAGNOSTIC
1277: cmpl $IPL_NONE,CPUVAR(ILEVEL)
1.142 maxv 1278: jne .Lspl_error
1.1 fvdl 1279: #endif
1.107 maxv 1280:
1.129 maxv 1281: /*
1.134 maxv 1282: * Decide if we need to take a slow path. That's the case when we
1283: * want to reload %cs and %ss on a 64bit LWP (MDL_IRET set), or when
1284: * we're returning to a 32bit LWP (MDL_COMPAT32 set).
1285: *
1286: * In either case, we jump into intrfastexit and return to userland
1287: * with the iret instruction.
1.129 maxv 1288: */
1.107 maxv 1289: testl $(MDL_IRET|MDL_COMPAT32),L_MD_FLAGS(%r14)
1.134 maxv 1290: jnz intrfastexit
1291:
1.148 maxv 1292: jmp syscall_sysret
1.1 fvdl 1293:
1294: #ifdef DIAGNOSTIC
1.107 maxv 1295: /* Report SPL error */
1.142 maxv 1296: .Lspl_error:
1.107 maxv 1297: movabsq $4f,%rdi
1.1 fvdl 1298: movl TF_RAX(%rsp),%esi
1299: movl TF_RDI(%rsp),%edx
1300: movl %ebx,%ecx
1301: movl CPUVAR(ILEVEL),%r8d
1302: xorq %rax,%rax
1303: call _C_LABEL(printf)
1.35 ad 1304: movl $IPL_NONE,%edi
1305: call _C_LABEL(spllower)
1.29 yamt 1306: jmp .Lsyscall_checkast
1.1 fvdl 1307: 4: .asciz "WARNING: SPL NOT LOWERED ON SYSCALL %d %d EXIT %x %x\n"
1308: #endif
1.70 dsl 1309:
1310: /* AST pending or pmap load needed */
1.41 ad 1311: 9:
1.107 maxv 1312: cmpl $0,CPUVAR(WANT_PMAPLOAD)
1.41 ad 1313: jz 10f
1314: STI(si)
1.29 yamt 1315: call _C_LABEL(do_pmap_load)
1316: jmp .Lsyscall_checkast /* re-check ASTs */
1.41 ad 1317: 10:
1318: CLEAR_ASTPENDING(%r14)
1319: STI(si)
1320: /* Pushed T_ASTFLT into tf_trapno on entry. */
1321: movq %rsp,%rdi
1322: call _C_LABEL(trap)
1323: jmp .Lsyscall_checkast /* re-check ASTs */
1.148 maxv 1324: END(handle_syscall)
1.29 yamt 1325:
1.22 yamt 1326: /*
1327: * void lwp_trampoline(void);
1328: *
1329: * This is a trampoline function pushed run by newly created LWPs
1.70 dsl 1330: * in order to do additional setup in their context.
1.22 yamt 1331: */
1332: NENTRY(lwp_trampoline)
1333: movq %rbp,%rsi
1.41 ad 1334: movq %rbp,%r14 /* for .Lsyscall_checkast */
1.22 yamt 1335: movq %rax,%rdi
1336: xorq %rbp,%rbp
1337: call _C_LABEL(lwp_startup)
1.1 fvdl 1338: movq %r13,%rdi
1339: call *%r12
1.41 ad 1340: jmp .Lsyscall_checkast
1.71 uebayasi 1341: END(lwp_trampoline)
1.1 fvdl 1342:
1343: /*
1.148 maxv 1344: * Entry points of the 'syscall' instruction, 64bit and 32bit mode.
1345: */
1346: TEXT_USER_BEGIN
1347:
1348: IDTVEC(syscall)
1349: #ifndef XEN
1350: /*
1351: * The user %rip is in %rcx and the user %rflags in %r11. The kernel %cs
1352: * and %ss are loaded, but nothing else is.
1353: *
1354: * The 'swapgs' instruction gives us access to cpu-specific memory where
1355: * we can save a user register and then read the LWP's kernel stack
1356: * pointer.
1357: *
1358: * This code doesn't seem to set %ds, this may not matter since it is
1359: * ignored in 64bit mode, OTOH the syscall instruction sets %ss and that
1360: * is ignored as well.
1361: */
1362: swapgs
1363:
1364: #ifdef SVS
1365: movq %rax,SVS_UTLS+UTLS_SCRATCH
1366: movq SVS_UTLS+UTLS_RSP0,%rax
1367: #define SP(x) (x)-(TF_SS+8)(%rax)
1368: #else
1369: movq %r15,CPUVAR(SCRATCH)
1370: movq CPUVAR(CURLWP),%r15
1371: movq L_PCB(%r15),%r15
1372: movq PCB_RSP0(%r15),%r15 /* LWP's kernel stack pointer */
1373: #define SP(x) (x)-(TF_SS+8)(%r15)
1374: #endif
1375:
1376: /* Make stack look like an 'int nn' frame */
1377: movq $(LSEL(LUDATA_SEL, SEL_UPL)),SP(TF_SS) /* user %ss */
1378: movq %rsp,SP(TF_RSP) /* user %rsp */
1379: movq %r11,SP(TF_RFLAGS) /* user %rflags */
1380: movq $(LSEL(LUCODE_SEL, SEL_UPL)),SP(TF_CS) /* user %cs */
1381: movq %rcx,SP(TF_RIP) /* user %rip */
1382:
1383: leaq SP(0),%rsp /* %rsp now valid after frame */
1384: #ifdef SVS
1385: movq SVS_UTLS+UTLS_SCRATCH,%rax
1386: #else
1387: movq CPUVAR(SCRATCH),%r15
1388: #endif
1389:
1390: movq $2,TF_ERR(%rsp) /* syscall instruction size */
1391: movq $T_ASTFLT,TF_TRAPNO(%rsp)
1392: #else
1393: /* Xen already switched to kernel stack */
1394: addq $0x10,%rsp /* gap to match cs:rip */
1395: pushq $2 /* error code */
1396: pushq $T_ASTFLT
1397: subq $TF_REGSIZE,%rsp
1398: cld
1399: #endif
1400: INTR_SAVE_GPRS
1401: movw $GSEL(GUDATA_SEL, SEL_UPL),TF_DS(%rsp)
1402: movw $GSEL(GUDATA_SEL, SEL_UPL),TF_ES(%rsp)
1403: movw $0,TF_FS(%rsp)
1404: movw $0,TF_GS(%rsp)
1405: SVS_ENTER
1406: jmp handle_syscall
1407: IDTVEC_END(syscall)
1408:
1409: IDTVEC(syscall32)
1410: sysret /* go away please */
1411: IDTVEC_END(syscall32)
1412:
1413: TEXT_USER_END
1414:
1415: /*
1.22 yamt 1416: * osyscall()
1417: *
1.1 fvdl 1418: * Trap gate entry for int $80 syscall, also used by sigreturn.
1419: */
1.148 maxv 1420: TEXT_USER_BEGIN
1.1 fvdl 1421: IDTVEC(osyscall)
1.32 bouyer 1422: #ifdef XEN
1423: movq (%rsp),%rcx
1424: movq 8(%rsp),%r11
1425: addq $0x10,%rsp
1426: #endif
1.84 maxv 1427: pushq $2 /* size of instruction for restart */
1428: pushq $T_ASTFLT /* trap # for doing ASTs */
1.1 fvdl 1429: INTRENTRY
1.148 maxv 1430: jmp handle_syscall
1.71 uebayasi 1431: IDTVEC_END(osyscall)
1.148 maxv 1432: TEXT_USER_END
1433:
1434: /*
1435: * Return to userland via 'sysret'.
1436: */
1437: TEXT_USER_BEGIN
1438: _ALIGN_TEXT
1439: LABEL(syscall_sysret)
1440: SVS_LEAVE
1441: INTR_RESTORE_GPRS
1442: SWAPGS
1443: #ifndef XEN
1444: movq TF_RIP(%rsp),%rcx /* %rip for sysret */
1445: movq TF_RFLAGS(%rsp),%r11 /* %flags for sysret */
1446: movq TF_RSP(%rsp),%rsp
1447: do_sysret:
1448: sysretq
1449: #else
1450: addq $TF_RIP,%rsp
1451: pushq $256 /* VGCF_IN_SYSCALL */
1452: jmp HYPERVISOR_iret
1453: #endif
1454: END(syscall_sysret)
1455: TEXT_USER_END
1.26 ad 1456:
1457: /*
1.66 chs 1458: * bool sse2_idlezero_page(void *pg)
1.26 ad 1459: *
1.47 ad 1460: * Zero a page without polluting the cache. Preemption must be
1461: * disabled by the caller. Abort if a preemption is pending.
1.66 chs 1462: * Returns true if the page is zeroed, false if not.
1.26 ad 1463: */
1.47 ad 1464: ENTRY(sse2_idlezero_page)
1465: pushq %rbp
1466: movq %rsp,%rbp
1467: movl $(PAGE_SIZE/64), %ecx
1.26 ad 1468: xorq %rax, %rax
1469: .align 16
1470: 1:
1.50 ad 1471: testl $RESCHED_KPREEMPT, CPUVAR(RESCHED)
1.47 ad 1472: jnz 2f
1.26 ad 1473: movnti %rax, 0(%rdi)
1474: movnti %rax, 8(%rdi)
1475: movnti %rax, 16(%rdi)
1476: movnti %rax, 24(%rdi)
1477: movnti %rax, 32(%rdi)
1478: movnti %rax, 40(%rdi)
1479: movnti %rax, 48(%rdi)
1480: movnti %rax, 56(%rdi)
1.47 ad 1481: addq $64, %rdi
1482: decl %ecx
1.26 ad 1483: jnz 1b
1484: sfence
1.47 ad 1485: incl %eax
1486: popq %rbp
1.26 ad 1487: ret
1.47 ad 1488: 2:
1.26 ad 1489: sfence
1.47 ad 1490: popq %rbp
1.26 ad 1491: ret
1.71 uebayasi 1492: END(sse2_idlezero_page)
1.66 chs 1493:
1494: /*
1495: * void pagezero(vaddr_t va)
1496: *
1497: * Zero a page without polluting the cache.
1498: */
1499: ENTRY(pagezero)
1500: movq $-PAGE_SIZE,%rdx
1501: subq %rdx,%rdi
1502: xorq %rax,%rax
1503: 1:
1504: movnti %rax,(%rdi,%rdx)
1505: movnti %rax,8(%rdi,%rdx)
1506: movnti %rax,16(%rdi,%rdx)
1507: movnti %rax,24(%rdi,%rdx)
1508: movnti %rax,32(%rdi,%rdx)
1509: movnti %rax,40(%rdi,%rdx)
1510: movnti %rax,48(%rdi,%rdx)
1511: movnti %rax,56(%rdi,%rdx)
1512: addq $64,%rdx
1513: jne 1b
1514: sfence
1515: ret
1.71 uebayasi 1516: END(pagezero)
1.129 maxv 1517:
1.148 maxv 1518: TEXT_USER_BEGIN
1519:
1520: _ALIGN_TEXT
1521: LABEL(intrfastexit)
1.147 maxv 1522: NOT_XEN(cli;)
1523: SVS_LEAVE
1.129 maxv 1524: INTR_RESTORE_GPRS
1.150 ! maxv 1525: testb $SEL_UPL,TF_CS(%rsp) /* interrupted %cs */
1.129 maxv 1526: jz .Lkexit
1.136 maxv 1527: cmpw $LSEL(LUCODE_SEL, SEL_UPL),TF_CS(%rsp)
1.134 maxv 1528: je .Luexit64
1.136 maxv 1529: cmpw $GSEL(GUCODE_SEL, SEL_UPL),TF_CS(%rsp)
1.134 maxv 1530: je .Luexit64
1.139 maxv 1531: #ifdef XEN
1.140 bouyer 1532: cmpw $FLAT_RING3_CS64,TF_CS(%rsp)
1.139 maxv 1533: je .Luexit64
1534: #endif
1.129 maxv 1535:
1.134 maxv 1536: .Luexit32:
1.138 maxv 1537: do_mov_es:
1.129 maxv 1538: movw TF_ES(%rsp),%es
1.138 maxv 1539: do_mov_ds:
1.129 maxv 1540: movw TF_DS(%rsp),%ds
1.138 maxv 1541: do_mov_fs:
1.135 maxv 1542: movw TF_FS(%rsp),%fs
1.129 maxv 1543: SWAPGS
1.137 maxv 1544: #ifndef XEN
1.138 maxv 1545: do_mov_gs:
1.137 maxv 1546: movw TF_GS(%rsp),%gs
1547: #endif
1.134 maxv 1548: jmp .Lkexit
1549:
1550: .Luexit64:
1551: SWAPGS
1.129 maxv 1552:
1553: .Lkexit:
1554: addq $TF_REGSIZE+16,%rsp /* + T_xxx and error code */
1555: do_iret:
1556: iretq
1557: END(intrfastexit)
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to locore.S CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / amd64 / amd64