Annotation of src/sys/arch/amd64/amd64/locore.S, Revision 1.196
1.196 ! ad 1: /* $NetBSD: locore.S,v 1.195 2019/12/15 02:58:21 manu Exp $ */
1.1 fvdl 2:
3: /*
4: * Copyright-o-rama!
5: */
6:
7: /*
1.84 maxv 8: * Copyright (c) 1998, 2000, 2007, 2008, 2016 The NetBSD Foundation, Inc.
9: * All rights reserved.
10: *
11: * This code is derived from software contributed to The NetBSD Foundation
1.100 maxv 12: * by Charles M. Hannum and by Maxime Villard.
1.84 maxv 13: *
14: * Redistribution and use in source and binary forms, with or without
15: * modification, are permitted provided that the following conditions
16: * are met:
17: * 1. Redistributions of source code must retain the above copyright
18: * notice, this list of conditions and the following disclaimer.
19: * 2. Redistributions in binary form must reproduce the above copyright
20: * notice, this list of conditions and the following disclaimer in the
21: * documentation and/or other materials provided with the distribution.
22: *
23: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
24: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
25: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
26: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
27: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33: * POSSIBILITY OF SUCH DAMAGE.
34: */
35:
36: /*
1.32 bouyer 37: * Copyright (c) 2007 Manuel Bouyer.
38: *
39: * Redistribution and use in source and binary forms, with or without
40: * modification, are permitted provided that the following conditions
41: * are met:
42: * 1. Redistributions of source code must retain the above copyright
43: * notice, this list of conditions and the following disclaimer.
44: * 2. Redistributions in binary form must reproduce the above copyright
45: * notice, this list of conditions and the following disclaimer in the
46: * documentation and/or other materials provided with the distribution.
47: *
48: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
49: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
50: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
51: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
52: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
53: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
54: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
55: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
56: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
57: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
58: *
59: */
60:
61: /*
62: * Copyright (c) 2006 Mathieu Ropert <mro@adviseo.fr>
63: *
64: * Permission to use, copy, modify, and distribute this software for any
65: * purpose with or without fee is hereby granted, provided that the above
66: * copyright notice and this permission notice appear in all copies.
67: *
68: * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
69: * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
70: * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
71: * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
72: * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
73: * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
74: * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
75: */
76:
77: /*
1.1 fvdl 78: * Copyright (c) 2001 Wasabi Systems, Inc.
79: * All rights reserved.
80: *
81: * Written by Frank van der Linden for Wasabi Systems, Inc.
82: *
83: * Redistribution and use in source and binary forms, with or without
84: * modification, are permitted provided that the following conditions
85: * are met:
86: * 1. Redistributions of source code must retain the above copyright
87: * notice, this list of conditions and the following disclaimer.
88: * 2. Redistributions in binary form must reproduce the above copyright
89: * notice, this list of conditions and the following disclaimer in the
90: * documentation and/or other materials provided with the distribution.
91: * 3. All advertising materials mentioning features or use of this software
92: * must display the following acknowledgement:
93: * This product includes software developed for the NetBSD Project by
94: * Wasabi Systems, Inc.
95: * 4. The name of Wasabi Systems, Inc. may not be used to endorse
96: * or promote products derived from this software without specific prior
97: * written permission.
98: *
99: * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
100: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
101: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
102: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
103: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
104: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
105: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
106: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
107: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
108: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
109: * POSSIBILITY OF SUCH DAMAGE.
110: */
111:
112: /*-
113: * Copyright (c) 1990 The Regents of the University of California.
114: * All rights reserved.
115: *
116: * This code is derived from software contributed to Berkeley by
117: * William Jolitz.
118: *
119: * Redistribution and use in source and binary forms, with or without
120: * modification, are permitted provided that the following conditions
121: * are met:
122: * 1. Redistributions of source code must retain the above copyright
123: * notice, this list of conditions and the following disclaimer.
124: * 2. Redistributions in binary form must reproduce the above copyright
125: * notice, this list of conditions and the following disclaimer in the
126: * documentation and/or other materials provided with the distribution.
1.5 agc 127: * 3. Neither the name of the University nor the names of its contributors
1.1 fvdl 128: * may be used to endorse or promote products derived from this software
129: * without specific prior written permission.
130: *
131: * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
132: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
133: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
134: * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
135: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
136: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
137: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
138: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
139: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
140: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
141: * SUCH DAMAGE.
142: *
143: * @(#)locore.s 7.3 (Berkeley) 5/13/91
144: */
145:
1.84 maxv 146: /* Override user-land alignment before including asm.h */
1.34 dsl 147: #define ALIGN_DATA .align 8
148: #define ALIGN_TEXT .align 16,0x90
149: #define _ALIGN_TEXT ALIGN_TEXT
150:
151: #include <machine/asm.h>
152:
1.78 uebayasi 153: #include "opt_copy_symtab.h"
1.1 fvdl 154: #include "opt_ddb.h"
155: #include "opt_ddbparam.h"
1.51 apb 156: #include "opt_modular.h"
1.1 fvdl 157: #include "opt_realmem.h"
158:
1.12 drochner 159: #include "opt_compat_netbsd.h"
160: #include "opt_compat_netbsd32.h"
1.193 manu 161: #include "opt_multiboot.h"
1.32 bouyer 162: #include "opt_xen.h"
1.145 maxv 163: #include "opt_svs.h"
1.12 drochner 164:
1.1 fvdl 165: #include "assym.h"
166: #include "lapic.h"
167: #include "ioapic.h"
1.2 fvdl 168: #include "ksyms.h"
1.1 fvdl 169:
170: #include <sys/errno.h>
171: #include <sys/syscall.h>
172:
173: #include <machine/pte.h>
174: #include <machine/segments.h>
175: #include <machine/specialreg.h>
176: #include <machine/trap.h>
177: #include <machine/bootinfo.h>
178: #include <machine/frameasm.h>
1.44 ad 179: #include <machine/cputypes.h>
1.1 fvdl 180:
1.193 manu 181: #ifndef XENPV
182: #include <arch/i386/include/multiboot.h>
183: #endif
184:
185: #define CODE_SEGMENT 0x08
186: #define DATA_SEGMENT 0x10
187:
1.1 fvdl 188: #if NLAPIC > 0
189: #include <machine/i82489reg.h>
190: #endif
191:
192: /* Get definitions for IOM_BEGIN, IOM_END, and IOM_SIZE */
193: #include <dev/isa/isareg.h>
194:
1.71 uebayasi 195: #define _RELOC(x) ((x) - KERNBASE)
196: #define RELOC(x) _RELOC(_C_LABEL(x))
197:
1.180 maxv 198: /* 32bit version of PTE_NX */
199: #define PTE_NX32 0x80000000
1.86 maxv 200:
1.83 maxv 201: #if L2_SLOT_KERNBASE > 0
202: #define TABLE_L2_ENTRIES (2 * (NKL2_KIMG_ENTRIES + 1))
203: #else
204: #define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1)
205: #endif
206:
207: #if L3_SLOT_KERNBASE > 0
208: #define TABLE_L3_ENTRIES (2 * NKL3_KIMG_ENTRIES)
209: #else
210: #define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES
211: #endif
212:
213: #define PROC0_PML4_OFF 0
1.97 maxv 214: #define PROC0_STK_OFF (PROC0_PML4_OFF + 1 * PAGE_SIZE)
215: #define PROC0_PTP3_OFF (PROC0_STK_OFF + UPAGES * PAGE_SIZE)
1.83 maxv 216: #define PROC0_PTP2_OFF (PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * PAGE_SIZE)
217: #define PROC0_PTP1_OFF (PROC0_PTP2_OFF + TABLE_L3_ENTRIES * PAGE_SIZE)
218: #define TABLESIZE \
1.97 maxv 219: ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \
1.83 maxv 220: * PAGE_SIZE)
221:
1.121 maxv 222: /* Amount of VA used to map the kernel, the syms and the preloaded modules */
223: #define BOOTMAP_VA_SIZE \
224: (NKL2_KIMG_ENTRIES * (1 << L2_SHIFT) - TABLESIZE - IOM_SIZE)
225:
1.83 maxv 226: /*
227: * fillkpt - Fill in a kernel page table
228: * eax = pte (page frame | control | status)
229: * ebx = page table address
230: * ecx = number of pages to map
231: *
232: * Each entry is 8 (PDE_SIZE) bytes long: we must set the 4 upper bytes to 0.
233: */
234: #define fillkpt \
1.94 maxv 235: cmpl $0,%ecx ; /* zero-sized? */ \
236: je 2f ; \
1.91 maxv 237: 1: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \
238: movl %eax,(%ebx) ; /* store phys addr */ \
239: addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
240: addl $PAGE_SIZE,%eax ; /* next phys page */ \
1.94 maxv 241: loop 1b ; \
242: 2: ;
1.83 maxv 243:
1.84 maxv 244: /*
1.89 maxv 245: * fillkpt_nox - Same as fillkpt, but sets the NX/XD bit.
246: */
247: #define fillkpt_nox \
1.94 maxv 248: cmpl $0,%ecx ; /* zero-sized? */ \
249: je 2f ; \
1.91 maxv 250: pushl %ebp ; \
251: movl RELOC(nox_flag),%ebp ; \
252: 1: movl %ebp,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: NX */ \
253: movl %eax,(%ebx) ; /* store phys addr */ \
1.89 maxv 254: addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
255: addl $PAGE_SIZE,%eax ; /* next phys page */ \
1.91 maxv 256: loop 1b ; \
1.94 maxv 257: popl %ebp ; \
258: 2: ;
1.89 maxv 259:
260: /*
1.96 maxv 261: * fillkpt_blank - Fill in a kernel page table with blank entries
262: * ebx = page table address
263: * ecx = number of pages to map
264: */
265: #define fillkpt_blank \
266: cmpl $0,%ecx ; /* zero-sized? */ \
267: je 2f ; \
268: 1: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \
269: movl $0,(%ebx) ; /* lower 32 bits: 0 */ \
270: addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
271: loop 1b ; \
272: 2: ;
273:
274: /*
1.84 maxv 275: * killkpt - Destroy a kernel page table (long mode)
276: * rbx = page table address
277: * rcx = number of pages to destroy
278: */
279: #define killkpt \
280: 1: movq $0,(%rbx) ; \
281: addq $PDE_SIZE,%rbx ; \
282: loop 1b ;
283:
1.83 maxv 284:
1.32 bouyer 285: #ifdef XEN
1.99 bouyer 286: #define __ASSEMBLY__
1.175 cherry 287: #include <xen/include/public/elfnote.h>
288: #include <xen/include/public/xen.h>
1.177 cherry 289: #endif /* XEN */
290:
1.176 cherry 291: #ifdef XENPV
1.99 bouyer 292: #define ELFNOTE(name, type, desctype, descdata...) \
293: .pushsection .note.name ; \
294: .align 4 ; \
295: .long 2f - 1f /* namesz */ ; \
296: .long 4f - 3f /* descsz */ ; \
297: .long type ; \
298: 1:.asciz #name ; \
299: 2:.align 4 ; \
300: 3:desctype descdata ; \
301: 4:.align 4 ; \
302: .popsection
303:
1.32 bouyer 304: /*
1.73 uebayasi 305: * Xen guest identifier and loader selection
1.32 bouyer 306: */
307: .section __xen_guest
1.99 bouyer 308: ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS, .asciz, "NetBSD")
309: ELFNOTE(Xen, XEN_ELFNOTE_GUEST_VERSION, .asciz, "4.99")
310: ELFNOTE(Xen, XEN_ELFNOTE_XEN_VERSION, .asciz, "xen-3.0")
311: ELFNOTE(Xen, XEN_ELFNOTE_VIRT_BASE, .quad, KERNBASE)
312: ELFNOTE(Xen, XEN_ELFNOTE_PADDR_OFFSET, .quad, KERNBASE)
313: ELFNOTE(Xen, XEN_ELFNOTE_ENTRY, .quad, start)
314: ELFNOTE(Xen, XEN_ELFNOTE_HYPERCALL_PAGE, .quad, hypercall_page)
315: ELFNOTE(Xen, XEN_ELFNOTE_HV_START_LOW, .quad, HYPERVISOR_VIRT_START)
316: ELFNOTE(Xen, XEN_ELFNOTE_FEATURES, .asciz, "")
317: ELFNOTE(Xen, XEN_ELFNOTE_PAE_MODE, .asciz, "yes")
1.180 maxv 318: ELFNOTE(Xen, XEN_ELFNOTE_L1_MFN_VALID, .long, PTE_P, PTE_P)\
1.99 bouyer 319: ELFNOTE(Xen, XEN_ELFNOTE_LOADER, .asciz, "generic")
320: ELFNOTE(Xen, XEN_ELFNOTE_SUSPEND_CANCEL, .long, 0)
1.49 ad 321: #if NKSYMS > 0 || defined(DDB) || defined(MODULAR)
1.99 bouyer 322: ELFNOTE(Xen, XEN_ELFNOTE_BSD_SYMTAB, .asciz, "yes")
1.32 bouyer 323: #endif
1.176 cherry 324: #endif /* XENPV */
1.1 fvdl 325:
326: /*
327: * Initialization
328: */
329: .data
330:
1.98 maxv 331: .globl _C_LABEL(tablesize)
1.86 maxv 332: .globl _C_LABEL(nox_flag)
1.100 maxv 333: .globl _C_LABEL(cputype)
1.57 jym 334: .globl _C_LABEL(cpuid_level)
1.71 uebayasi 335: .globl _C_LABEL(esym)
336: .globl _C_LABEL(eblob)
1.100 maxv 337: .globl _C_LABEL(atdevbase)
338: .globl _C_LABEL(PDPpaddr)
1.71 uebayasi 339: .globl _C_LABEL(boothowto)
340: .globl _C_LABEL(bootinfo)
341: .globl _C_LABEL(biosbasemem)
342: .globl _C_LABEL(biosextmem)
1.130 maxv 343: .globl _C_LABEL(lwp0uarea)
1.138 maxv 344: .globl do_mov_es
345: .globl do_mov_ds
346: .globl do_mov_fs
347: .globl do_mov_gs
1.129 maxv 348: .globl do_iret
1.43 ad 349:
1.98 maxv 350: .type _C_LABEL(tablesize), @object
351: _C_LABEL(tablesize): .long TABLESIZE
352: END(tablesize)
1.86 maxv 353: .type _C_LABEL(nox_flag), @object
354: LABEL(nox_flag) .long 0 /* 32bit NOX flag, set if supported */
355: END(nox_flag)
1.71 uebayasi 356: .type _C_LABEL(cputype), @object
1.84 maxv 357: LABEL(cputype) .long 0 /* are we 80486, Pentium, or.. */
1.71 uebayasi 358: END(cputype)
359: .type _C_LABEL(cpuid_level), @object
1.84 maxv 360: LABEL(cpuid_level) .long -1 /* max. level accepted by cpuid instr */
1.71 uebayasi 361: END(cpuid_level)
362: .type _C_LABEL(esym), @object
1.84 maxv 363: LABEL(esym) .quad 0 /* ptr to end of syms */
1.71 uebayasi 364: END(esym)
365: .type _C_LABEL(eblob), @object
1.84 maxv 366: LABEL(eblob) .quad 0 /* ptr to end of modules */
1.71 uebayasi 367: END(eblob)
368: .type _C_LABEL(atdevbase), @object
1.84 maxv 369: LABEL(atdevbase) .quad 0 /* location of start of iomem in virt */
1.71 uebayasi 370: END(atdevbase)
371: .type _C_LABEL(PDPpaddr), @object
1.84 maxv 372: LABEL(PDPpaddr) .quad 0 /* paddr of PTD, for libkvm */
1.71 uebayasi 373: END(PDPpaddr)
374: .type _C_LABEL(biosbasemem), @object
1.1 fvdl 375: #ifndef REALBASEMEM
1.84 maxv 376: LABEL(biosbasemem) .long 0 /* base memory reported by BIOS */
1.1 fvdl 377: #else
1.71 uebayasi 378: LABEL(biosbasemem) .long REALBASEMEM
1.1 fvdl 379: #endif
1.71 uebayasi 380: END(biosbasemem)
381: .type _C_LABEL(biosextmem), @object
1.1 fvdl 382: #ifndef REALEXTMEM
1.84 maxv 383: LABEL(biosextmem) .long 0 /* extended memory reported by BIOS */
1.1 fvdl 384: #else
1.71 uebayasi 385: LABEL(biosextmem) .long REALEXTMEM
1.1 fvdl 386: #endif
1.71 uebayasi 387: END(biosextmem)
1.130 maxv 388: .type _C_LABEL(lwp0uarea), @object
389: LABEL(lwp0uarea) .quad 0
390: END(lwp0uarea)
1.1 fvdl 391:
1.176 cherry 392: #ifndef XENPV
1.56 jym 393: .globl gdt64_lo
394: .globl gdt64_hi
1.1 fvdl 395:
1.56 jym 396: #define GDT64_LIMIT gdt64_end-gdt64_start-1
397: /* Temporary gdt64, with base address in low memory */
1.71 uebayasi 398: .type _C_LABEL(gdt64_lo), @object
399: LABEL(gdt64_lo)
1.56 jym 400: .word GDT64_LIMIT
1.1 fvdl 401: .quad _RELOC(gdt64_start)
1.71 uebayasi 402: END(gdt64_lo)
1.1 fvdl 403: .align 64
404:
1.56 jym 405: /* Temporary gdt64, with base address in high memory */
1.71 uebayasi 406: .type _C_LABEL(gdt64_hi), @object
407: LABEL(gdt64_hi)
1.56 jym 408: .word GDT64_LIMIT
409: .quad gdt64_start
1.71 uebayasi 410: END(gdt64_hi)
1.56 jym 411: .align 64
412: #undef GDT64_LIMIT
413:
1.71 uebayasi 414: .type _C_LABEL(gdt64_start), @object
415: _C_LABEL(gdt64_start):
1.1 fvdl 416: .quad 0x0000000000000000 /* always empty */
417: .quad 0x00af9a000000ffff /* kernel CS */
418: .quad 0x00cf92000000ffff /* kernel DS */
1.71 uebayasi 419: END(gdt64_start)
1.1 fvdl 420: gdt64_end:
421:
1.71 uebayasi 422: .type _C_LABEL(farjmp64), @object
423: _C_LABEL(farjmp64):
1.57 jym 424: .long _RELOC(longmode)
1.1 fvdl 425: .word GSEL(GCODE_SEL, SEL_KPL)
1.71 uebayasi 426: END(farjmp64)
1.84 maxv 427:
1.176 cherry 428: #endif /* !XENPV */
1.71 uebayasi 429:
1.84 maxv 430: /* Space for the temporary stack */
1.71 uebayasi 431: .size tmpstk, tmpstk - .
1.84 maxv 432: .space 512
1.1 fvdl 433: tmpstk:
434:
1.194 manu 435: .section multiboot,"a"
1.193 manu 436: #if defined(MULTIBOOT)
437: .align 8
438: .globl Multiboot2_Header
439: _C_LABEL(Multiboot2_Header):
440: .int MULTIBOOT2_HEADER_MAGIC
441: .int MULTIBOOT2_ARCHITECTURE_I386
442: .int Multiboot2_Header_end - Multiboot2_Header
443: .int -(MULTIBOOT2_HEADER_MAGIC + MULTIBOOT2_ARCHITECTURE_I386 \
444: + (Multiboot2_Header_end - Multiboot2_Header))
445:
446: .int 1 /* MULTIBOOT_HEADER_TAG_INFORMATION_REQUEST */
447: .int 12 /* sizeof(multiboot_header_tag_information_request) */
448: /* + sizeof(uint32_t) * requests */
449: .int 4 /* MULTIBOOT_TAG_TYPE_BASIC_MEMINFO */
450: .align 8
451:
452: .int 3 /* MULTIBOOT_HEADER_TAG_ENTRY_ADDRESS */
453: .int 16 /* sizeof(struct multiboot_tag_efi64) */
454: .quad (multiboot2_entry - KERNBASE)
455: .align 8
456:
457: .int 9 /* MULTIBOOT_HEADER_TAG_ENTRY_ADDRESS_EFI64 */
458: .int 16 /* sizeof(struct multiboot_tag_efi64) */
459: .quad (multiboot2_entry - KERNBASE)
460: .align 8
461:
462: #if notyet
463: /*
464: * Could be used to get an early console for debug,
465: * but this is broken.
466: */
467: .int 7 /* MULTIBOOT_HEADER_TAG_EFI_BS */
468: .int 8 /* sizeof(struct multiboot_tag) */
469: .align 8
470: #endif
471:
472: .int 0 /* MULTIBOOT_HEADER_TAG_END */
473: .int 8 /* sizeof(struct multiboot_tag) */
474: .align 8
475: .globl Multiboot2_Header_end
476: _C_LABEL(Multiboot2_Header_end):
477: #endif /* MULTIBOOT */
478:
1.1 fvdl 479: /*
480: * Some hackage to deal with 64bit symbols in 32 bit mode.
1.79 maxv 481: * This may not be needed if things are cleaned up a little.
1.1 fvdl 482: */
483:
484: .text
485: .globl _C_LABEL(kernel_text)
486: .set _C_LABEL(kernel_text),KERNTEXTOFF
487:
1.71 uebayasi 488: ENTRY(start)
1.176 cherry 489: #ifndef XENPV
1.71 uebayasi 490: .code32
1.81 maxv 491:
492: /* Warm boot */
493: movw $0x1234,0x472
494:
1.193 manu 495: #if defined(MULTIBOOT)
496: jmp .Lnative_loader
497:
498:
499: multiboot2_entry:
500: .code64
501: /*
502: * multiboot2 entry point. We are left here without
503: * stack and with no idea of where we were loaded in memory.
504: * The only inputs are
505: * %eax MULTIBOOT2_BOOTLOADER_MAGIC
506: * %ebx pointer to multiboot_info
507: *
508: * Here we will:
509: * - copy the kernel to 0x200000 (KERNTEXTOFF - KERNBASE)
510: * as almost all the code in locore.S assume it is there.
511: * This is derived from
512: * src/sys/arch/i386/stand/efiboot/bootx64/startprog64.S
513: * - copy multiboot_info, as done in multiboot_pre_reloc() from
514: * src/sys/arch/x86/x86/multiboot2.c
515: * Unfortunately we cannot call that function as there is
516: * no simple way to build it as 32 bit code in a 64 bit kernel.
517: * - Copy ELF symbols, also as in multiboot_pre_reloc()
518: */
519:
520: cli
521:
522: /*
523: * Discover our load address and use it to get start address
524: */
525: mov $_RELOC(tmpstk),%rsp
526: call next
527: next: pop %r8
528: sub $(next - start), %r8
529:
530: /*
531: * Save multiboot_info for later. We cannot use
532: * temporary stack for that since we are going to
533: * overwrite it.
534: */
535: movl %ebx, (multiboot2_info_ptr - start)(%r8)
536:
537: /*
538: * Get relocated multiboot2_loader entry point in %r9
539: */
540: mov $(KERNTEXTOFF - KERNBASE), %r9
541: add $(multiboot2_loader - kernel_text), %r9
542:
543: /* Copy kernel */
544: mov $(KERNTEXTOFF - KERNBASE), %rdi /* dest */
545: mov %r8, %rsi
546: sub $(start - kernel_text), %rsi /* src */
1.195 manu 547: mov $(__kernel_end - kernel_text), %rcx /* size */
1.193 manu 548: mov %rcx, %r12
549: movq %rdi, %r11 /* for misaligned check */
550:
551: #if !defined(NO_OVERLAP)
552: movq %rdi, %r13
553: subq %rsi, %r13
554: #endif
555:
556: shrq $3, %rcx /* count for copy by words */
557: jz 8f /* j if less than 8 bytes */
558:
559: lea -8(%rdi, %r12), %r14 /* target address of last 8 */
560: mov -8(%rsi, %r12), %r15 /* get last word */
561: #if !defined(NO_OVERLAP)
562: cmpq %r12, %r13 /* overlapping? */
563: jb 10f
564: #endif
565:
566: /*
567: * Non-overlaping, copy forwards.
568: * Newer Intel cpus (Nehalem) will do 16byte read/write transfers
569: * if %ecx is more than 76.
570: * AMD might do something similar some day.
571: */
572: and $7, %r11 /* destination misaligned ? */
573: jnz 12f
574: rep
575: movsq
576: mov %r15, (%r14) /* write last word */
577: jmp .Lcopy_done
578:
579: /*
580: * Destination misaligned
581: * AMD say it is better to align the destination (not the source).
582: * This will also re-align copies if the source and dest are both
583: * misaligned by the same amount)
584: * (I think Nehalem will use its accelerated copy if the source
585: * and destination have the same alignment.)
586: */
587: 12:
588: lea -9(%r11, %r12), %rcx /* post re-alignment count */
589: neg %r11 /* now -1 .. -7 */
590: mov (%rsi), %r12 /* get first word */
591: mov %rdi, %r13 /* target for first word */
592: lea 8(%rsi, %r11), %rsi
593: lea 8(%rdi, %r11), %rdi
594: shr $3, %rcx
595: rep
596: movsq
597: mov %r12, (%r13) /* write first word */
598: mov %r15, (%r14) /* write last word */
599: jmp .Lcopy_done
600:
601: #if !defined(NO_OVERLAP)
602: /* Must copy backwards.
603: * Reverse copy is probably easy to code faster than 'rep movds'
604: * since that requires (IIRC) an extra clock every 3 iterations (AMD).
605: * However I don't suppose anything cares that much!
606: * The big cost is the std/cld pair - reputedly 50+ cycles on Netburst P4.
607: * The copy is aligned with the buffer start (more likely to
608: * be a multiple of 8 than the end).
609: */
610: 10:
611: lea -8(%rsi, %rcx, 8), %rsi
612: lea -8(%rdi, %rcx, 8), %rdi
613: std
614: rep
615: movsq
616: cld
617: mov %r15, (%r14) /* write last bytes */
618: jmp .Lcopy_done
619: #endif
620:
621: /* Less than 8 bytes to copy, copy by bytes */
622: /* Intel Nehalem optimise 'rep movsb' for <= 7 bytes (9-15 clocks).
623: * For longer transfers it is 50+ !
624: */
625: 8: mov %r12, %rcx
626:
627: #if !defined(NO_OVERLAP)
628: cmpq %r12, %r13 /* overlapping? */
629: jb 81f
630: #endif
631:
632: /* nope, copy forwards. */
633: rep
634: movsb
635: jmp .Lcopy_done
636:
637: #if !defined(NO_OVERLAP)
638: /* Must copy backwards */
639: 81:
640: lea -1(%rsi, %rcx), %rsi
641: lea -1(%rdi, %rcx), %rdi
642: std
643: rep
644: movsb
645: cld
646: #endif
647: /* End of copy kernel */
648: .Lcopy_done:
649:
650: mov %r8, %rdi /* %rdi: loaded start address */
651: mov %r9, %rsi /* %rsi: kernel entry address */
652:
653: /* Prepare jump address */
654: lea (multiboot2_loader32a - start)(%rdi), %rax
655: movl %eax, (multiboot2_loader32r - start)(%rdi)
656:
657: /* Setup GDT */
658: lea (gdt - start)(%rdi), %rax
659: mov %rax, (gdtrr - start)(%rdi)
660: lgdt (gdtr - start)(%rdi)
661:
662: /* Jump to set %cs */
663: ljmp *(multiboot2_loader32r - start)(%rdi)
664:
665: .align 4
666: .code32
667: multiboot2_loader32a:
668: movl $DATA_SEGMENT, %eax
669: movw %ax, %ds
670: movw %ax, %es
671: movw %ax, %fs
672: movw %ax, %gs
673: movw %ax, %ss
674:
675: /* Already set new stack pointer */
676: movl %esp, %ebp
677:
678: /* Disable Paging in CR0 */
679: movl %cr0, %eax
680: andl $(~CR0_PG), %eax
681: movl %eax, %cr0
682:
683: /* Disable PAE in CR4 */
684: movl %cr4, %eax
685: andl $(~CR4_PAE), %eax
686: movl %eax, %cr4
687:
688: jmp multiboot2_loader32b
689:
690: .align 4
691: multiboot2_loader32b:
692: xor %eax, %eax
693:
694: /*
695: * Reload multiboot info from target location
696: */
697: movl _RELOC(multiboot2_info_ptr), %ebx
698: call *%esi
699:
700: .align 16
701: multiboot2_loader32r:
702: .long 0
703: .long CODE_SEGMENT
704: .align 16
705: gdt:
706: .long 0, 0
707: .byte 0xff, 0xff, 0x00, 0x00, 0x00, 0x9f, 0xcf, 0x00
708: .byte 0xff, 0xff, 0x00, 0x00, 0x00, 0x93, 0xcf, 0x00
709: gdtr:
710: .word gdtr - gdt
711: gdtrr:
712: .quad 0
713:
714: multiboot2_info_ptr:
715: .long 0
716:
717: .align 16
718: multiboot2_loader:
719: /*
720: * Here we would like to call multiboot2_pre_reloc() but
721: * we do not yet run in long mode, which means we need
722: * a 32 bit version of that function. Unfortunately,
723: * mixing 32-bit and 64-bit object file at link time
724: * does not work. As a result, we need to do the job
725: * of multiboot2_pre_reloc() here in assembly.
726: */
727: #if multiboot2_pre_reloc_would_be_built_as_ia32
728: movl $_RELOC(tmpstk),%esp
729: mov %ebx,%edi /* Address of Multiboot information */
730: call _C_LABEL(multiboot2_pre_reloc)
731: #else
732: /*
733: * Copy multiboot_info
734: */
735: movl $_RELOC(multiboot_info),%edi
736: movl %ebx,%esi
737: movl (%ebx),%ecx
738: shr $2,%ecx
739: rep
740: movsl
741:
742: /*
743: * Set multiboot2_enabled
744: */
745: movl $1,%eax
746: movl %eax,RELOC(multiboot2_enabled)
747:
748: /*
749: * Look for MULTIBOOT_TAG_TYPE_ELF_SECTIONS
750: */
751: movl $_RELOC(multiboot_info),%esi
752: movl (%esi),%ecx /* multiboot_info size */
753: movl %esi,%edx
754: addl %ecx,%edx /* %edx: end of multiboot_info */
755: addl $8,%esi /* skip two words of multiboot_info header */
756: mbt_loop:
757: movl (%esi),%ebx /* mbt->type */
758: cmpl $9,%ebx /* 9 for MULTIBOOT_TAG_TYPE_ELF_SECTIONS */
759: je found_elf_sections
760:
761: movl 4(%esi),%eax /* mbt->size */
762: addl %eax,%esi
763: addl $7,%esi /* roundup(%esi,8) */
764: andl $~7,%esi
765:
766: cmpl %edx,%esi
767: jle mbt_loop
768: jmp elf_sections_done
769:
770: found_elf_sections:
771: movl $0,%eax
772: movl %esp,%ebp /* %ebp is esymp */
773: push %eax
774: push $KERNBASE_LO /* kernbase */
775: push $_RELOC(end) /* void *end */
776: push %ebp /* int **esymp */
777: push $_RELOC(has_syms) /* bool *has_symsp */
778: push $_RELOC(Multiboot_Symbols)/* struct multiboot_symbol *ms */
779: push %esi /* struct multiboot_tag_elf_sections *mbt_elf */
780: call multiboot2_copy_syms32
781:
782: /* Asjust esym as a 64 bit pointer if esymp was set */
783: movl (%ebp),%eax
784: testl %eax,%eax /* esymp = NULL? */
785: jz elf_sections_done
786:
787: movl $RELOC(esym),%ebp
788: movl %eax,(%ebp)
789: movl $KERNBASE_HI,4(%ebp)
790:
791: jmp elf_sections_done
792:
793: /*
794: * This is multiboot2_copy_syms() from
795: * src/sys/arch/x86/x86/multiboot2.c
796: * built with -m32 -mcmodel=32 -D_LOCORE_64
797: */
798: multiboot2_copy_syms32:
799: push %ebp
800: mov %esp,%ebp
801: push %edi
802: push %esi
803: push %ebx
804: sub $0x20,%esp
805: mov 0x8(%ebp),%esi
806: /* for (i = 0; i < mbt_elf->num && symtabp == NULL && */
807: mov 0x8(%esi),%ebx
808: test %ebx,%ebx
809: je copy_syms_4ce
810: add $0x14,%esi
811: mov %esi,%eax
812: xor %edx,%edx
813: jmp copy_syms_3a0
814: copy_syms_395:
815: cmp %edx,%ebx
816: jbe copy_syms_4ce
817: copy_syms_39d:
818: add $0x40,%eax
819: copy_syms_3a0:
820: add $0x1,%edx
821: /* if ((shdrp->sh_type == SHT_SYMTAB) && */
822: cmpl $0x2,0x4(%eax)
823: jne copy_syms_395
824: /* shdrp->sh_link != SHN_UNDEF) { */
825: mov 0x28(%eax),%ecx
826: /* if ((shdrp->sh_type == SHT_SYMTAB) && */
827: test %ecx,%ecx
828: je copy_syms_395
829: /* [shdrp->sh_link]; */
830: shl $0x6,%ecx
831: /* shdrp2 = &((locore_Elf_Shdr *)mbt_elf->sections) */
832: add %esi,%ecx
833: /* if (shdrp2->sh_type == SHT_STRTAB) { */
834: cmpl $0x3,0x4(%ecx)
835: jne copy_syms_395
836: /* for (i = 0; i < mbt_elf->num && symtabp == NULL && */
837: cmp %ebx,%edx
838: jae copy_syms_6d1
839: test %eax,%eax
840: je copy_syms_608
841: /* if (symtabp == NULL || strtabp == NULL) */
842: copy_syms_3cb:
843: test %ecx,%ecx
844: lea 0x0(%esi),%esi
845: je copy_syms_4ce
846: /* symaddr = symtabp->sh_addr; */
847: mov 0x10(%eax),%edi
848: mov %edi,-0x10(%ebp)
849: mov 0x14(%eax),%ebx
850: mov %ebx,-0x18(%ebp)
851: /* straddr = strtabp->sh_addr; */
852: mov 0x10(%ecx),%esi
853: mov %esi,-0x14(%ebp)
854: mov 0x14(%ecx),%ebx
855: mov %ebx,-0x20(%ebp)
856: /* symsize = symtabp->sh_size; */
857: mov 0x20(%eax),%ebx
858: /* strsize = strtabp->sh_size; */
859: mov 0x20(%ecx),%eax
860: mov %eax,-0x1c(%ebp)
861: cmp 0x18(%ebp),%edi
862: jae copy_syms_4d6
863: cmp %esi,0x18(%ebp)
864: ja copy_syms_4e0
865: jae copy_syms_54d
866: /* cp2dst = (locore_Elf_Addr)(uintptr_t)endp + cp1size; */
867: copy_syms_40f:
868: mov -0x1c(%ebp),%ecx
869: mov %ecx,%eax
870: xor %edx,%edx
871: /* cp1dst = (locore_Elf_Addr)(uintptr_t)endp; */
872: mov 0x18(%ebp),%esi
873: xor %edi,%edi
874: /* cp2dst = (locore_Elf_Addr)(uintptr_t)endp + cp1size; */
875: add %esi,%eax
876: adc %edi,%edx
877: mov %eax,-0x2c(%ebp)
878: mov %edx,-0x28(%ebp)
879: /* (void)memcpy((void *)(uintptr_t)cp1dst, */
880: mov %ecx,%eax
881: mov 0x18(%ebp),%edi
882: mov -0x14(%ebp),%esi
883: cmp $0x4,%ecx
884: jae copy_syms_5e8
885: copy_syms_436:
886: test $0x2,%al
887: je copy_syms_43c
888: movsw %ds:(%esi),%es:(%edi)
889: copy_syms_43c:
890: test $0x1,%al
891: je copy_syms_441
892: movsb %ds:(%esi),%es:(%edi)
893: /* (void)memcpy((void *)(uintptr_t)cp2dst, */
894: copy_syms_441:
895: mov %ebx,%eax
896: mov 0x18(%ebp),%edi
897: mov -0x1c(%ebp),%esi
898: add %esi,%edi
899: mov -0x10(%ebp),%esi
900: cmp $0x4,%ebx
901: jae copy_syms_5c4
902: copy_syms_457:
903: test $0x2,%al
904: je copy_syms_45d
905: movsw %ds:(%esi),%es:(%edi)
906: copy_syms_45d:
907: test $0x1,%al
908: je copy_syms_462
909: movsb %ds:(%esi),%es:(%edi)
910: /* symstart = (cp1src == symaddr) ? cp1dst : cp2dst; */
911: copy_syms_462:
912: mov -0x18(%ebp),%edx
913: mov -0x20(%ebp),%edi
914: xor %edi,%edx
915: mov -0x10(%ebp),%eax
916: mov -0x14(%ebp),%ecx
917: xor %ecx,%eax
918: or %eax,%edx
919: je copy_syms_6ba
920: mov -0x2c(%ebp),%eax
921: mov %eax,-0x24(%ebp)
922: mov %ecx,-0x10(%ebp)
923: mov %edi,-0x18(%ebp)
924: /* strstart = (cp1src == straddr) ? cp1dst : cp2dst; */
925: copy_syms_486:
926: mov -0x20(%ebp),%edx
927: xor -0x18(%ebp),%edx
928: mov -0x14(%ebp),%eax
929: xor -0x10(%ebp),%eax
930: or %eax,%edx
931: je copy_syms_545
932: copy_syms_49a:
933: mov -0x2c(%ebp),%esi
934: /* ms->s_symstart = symstart + kernbase; */
935: copy_syms_49d:
936: mov -0x24(%ebp),%eax
937: add 0x1c(%ebp),%eax
938: mov 0xc(%ebp),%edi
939: mov %eax,(%edi)
940: /* ms->s_symsize = symsize; */
941: mov %edi,%eax
942: mov %ebx,0x4(%edi)
943: /* ms->s_strstart = strstart + kernbase; */
944: add 0x1c(%ebp),%esi
945: mov %esi,0x8(%edi)
946: /* ms->s_strsize = strsize; */
947: mov -0x1c(%ebp),%edi
948: mov %edi,0xc(%eax)
949: /* *has_symsp = true; */
950: mov 0x10(%ebp),%eax
951: movb $0x1,(%eax)
952: /* *esymp = (int *)((uintptr_t)endp + symsize + strsize + kernbase); */
953: mov 0x18(%ebp),%eax
954: add 0x1c(%ebp),%eax
955: add %eax,%ebx
956: add %edi,%ebx
957: mov 0x14(%ebp),%eax
958: mov %ebx,(%eax)
959: copy_syms_4ce:
960: add $0x20,%esp
961: pop %ebx
962: pop %esi
963: pop %edi
964: pop %ebp
965: ret
966: copy_syms_4d6:
967: jbe copy_syms_54d
968: mov -0x14(%ebp),%eax
969: cmp %eax,0x18(%ebp)
970: jbe copy_syms_54d
971: /* cp1dst = (locore_Elf_Addr)(uintptr_t)endp; */
972: copy_syms_4e0:
973: mov 0x18(%ebp),%eax
974: mov %eax,-0x24(%ebp)
975: /* cp2dst = (locore_Elf_Addr)(uintptr_t)endp + cp1size; */
976: mov %ebx,%eax
977: xor %edx,%edx
978: /* cp1dst = (locore_Elf_Addr)(uintptr_t)endp; */
979: mov 0x18(%ebp),%esi
980: xor %edi,%edi
981: /* cp2dst = (locore_Elf_Addr)(uintptr_t)endp + cp1size; */
982: add %esi,%eax
983: adc %edi,%edx
984: mov %eax,-0x2c(%ebp)
985: mov %edx,-0x28(%ebp)
986: /* (void)memcpy((void *)(uintptr_t)cp1dst, */
987: mov %ebx,%eax
988: mov 0x18(%ebp),%edi
989: mov -0x10(%ebp),%esi
990: cmp $0x4,%ebx
991: jae copy_syms_5a8
992: copy_syms_50a:
993: test $0x2,%al
994: jne copy_syms_57b
995: test $0x1,%al
996: jne copy_syms_578
997: /* (void)memcpy((void *)(uintptr_t)cp2dst, */
998: copy_syms_512:
999: mov -0x1c(%ebp),%ecx
1000: mov %ecx,%eax
1001: mov 0x18(%ebp),%edi
1002: add %ebx,%edi
1003: mov -0x14(%ebp),%esi
1004: cmp $0x4,%ecx
1005: jae copy_syms_584
1006: copy_syms_524:
1007: test $0x2,%al
1008: jne copy_syms_56c
1009: test $0x1,%al
1010: je copy_syms_486
1011: copy_syms_530:
1012: movsb %ds:(%esi),%es:(%edi)
1013: /* strstart = (cp1src == straddr) ? cp1dst : cp2dst; */
1014: mov -0x20(%ebp),%edx
1015: xor -0x18(%ebp),%edx
1016: mov -0x14(%ebp),%eax
1017: xor -0x10(%ebp),%eax
1018: or %eax,%edx
1019: jne copy_syms_49a
1020: copy_syms_545:
1021: mov 0x18(%ebp),%esi
1022: jmp copy_syms_49d
1023: /* if (symaddr < straddr) { */
1024: copy_syms_54d:
1025: mov -0x20(%ebp),%edi
1026: cmp %edi,-0x18(%ebp)
1027: jb copy_syms_4e0
1028: ja copy_syms_40f
1029: mov -0x14(%ebp),%edi
1030: cmp %edi,-0x10(%ebp)
1031: jb copy_syms_4e0
1032: jmp copy_syms_40f
1033: /* (void)memcpy((void *)(uintptr_t)cp2dst, */
1034: copy_syms_56c:
1035: movsw %ds:(%esi),%es:(%edi)
1036: test $0x1,%al
1037: je copy_syms_486
1038: jmp copy_syms_530
1039: /* (void)memcpy((void *)(uintptr_t)cp1dst, */
1040: copy_syms_578:
1041: movsb %ds:(%esi),%es:(%edi)
1042: jmp copy_syms_512
1043: copy_syms_57b:
1044: movsw %ds:(%esi),%es:(%edi)
1045: test $0x1,%al
1046: nop
1047: je copy_syms_512
1048: jmp copy_syms_578
1049: /* (void)memcpy((void *)(uintptr_t)cp2dst, */
1050: copy_syms_584:
1051: test $0x1,%edi
1052: jne copy_syms_650
1053: copy_syms_590:
1054: test $0x2,%edi
1055: jne copy_syms_63c
1056: copy_syms_59c:
1057: mov %eax,%ecx
1058: shr $0x2,%ecx
1059: rep movsl %ds:(%esi),%es:(%edi)
1060: jmp copy_syms_524
1061: /* (void)memcpy((void *)(uintptr_t)cp1dst, */
1062: copy_syms_5a8:
1063: test $0x1,%edi
1064: jne copy_syms_626
1065: copy_syms_5b0:
1066: test $0x2,%edi
1067: jne copy_syms_615
1068: copy_syms_5b8:
1069: mov %eax,%ecx
1070: shr $0x2,%ecx
1071: rep movsl %ds:(%esi),%es:(%edi)
1072: jmp copy_syms_50a
1073: /* (void)memcpy((void *)(uintptr_t)cp2dst, */
1074: copy_syms_5c4:
1075: test $0x1,%edi
1076: jne copy_syms_666
1077: copy_syms_5d0:
1078: test $0x2,%edi
1079: jne copy_syms_6a6
1080: copy_syms_5dc:
1081: mov %eax,%ecx
1082: shr $0x2,%ecx
1083: rep movsl %ds:(%esi),%es:(%edi)
1084: jmp copy_syms_457
1085: /* (void)memcpy((void *)(uintptr_t)cp1dst, */
1086: copy_syms_5e8:
1087: test $0x1,%edi
1088: jne copy_syms_68d
1089: copy_syms_5f4:
1090: test $0x2,%edi
1091: jne copy_syms_679
1092: copy_syms_5fc:
1093: mov %eax,%ecx
1094: shr $0x2,%ecx
1095: rep movsl %ds:(%esi),%es:(%edi)
1096: jmp copy_syms_436
1097: /* for (i = 0; i < mbt_elf->num && symtabp == NULL && */
1098: copy_syms_608:
1099: test %ecx,%ecx
1100: jne copy_syms_4ce
1101: jmp copy_syms_39d
1102: /* (void)memcpy((void *)(uintptr_t)cp1dst, */
1103: copy_syms_615:
1104: movzwl (%esi),%edx
1105: mov %dx,(%edi)
1106: add $0x2,%edi
1107: add $0x2,%esi
1108: sub $0x2,%eax
1109: jmp copy_syms_5b8
1110: copy_syms_626:
1111: movzbl (%esi),%eax
1112: mov %al,(%edi)
1113: mov 0x18(%ebp),%eax
1114: lea 0x1(%eax),%edi
1115: add $0x1,%esi
1116: lea -0x1(%ebx),%eax
1117: jmp copy_syms_5b0
1118: /* (void)memcpy((void *)(uintptr_t)cp2dst, */
1119: copy_syms_63c:
1120: movzwl (%esi),%edx
1121: mov %dx,(%edi)
1122: add $0x2,%edi
1123: add $0x2,%esi
1124: sub $0x2,%eax
1125: jmp copy_syms_59c
1126: copy_syms_650:
1127: movzbl (%esi),%eax
1128: mov %al,(%edi)
1129: add $0x1,%edi
1130: add $0x1,%esi
1131: mov -0x1c(%ebp),%eax
1132: sub $0x1,%eax
1133: jmp copy_syms_590
1134: copy_syms_666:
1135: movzbl (%esi),%eax
1136: mov %al,(%edi)
1137: add $0x1,%edi
1138: add $0x1,%esi
1139: lea -0x1(%ebx),%eax
1140: jmp copy_syms_5d0
1141: /* (void)memcpy((void *)(uintptr_t)cp1dst, */
1142: copy_syms_679:
1143: movzwl (%esi),%edx
1144: mov %dx,(%edi)
1145: add $0x2,%edi
1146: add $0x2,%esi
1147: sub $0x2,%eax
1148: jmp copy_syms_5fc
1149: copy_syms_68d:
1150: movzbl (%esi),%eax
1151: mov %al,(%edi)
1152: mov 0x18(%ebp),%eax
1153: lea 0x1(%eax),%edi
1154: add $0x1,%esi
1155: mov -0x1c(%ebp),%eax
1156: sub $0x1,%eax
1157: jmp copy_syms_5f4
1158: /* (void)memcpy((void *)(uintptr_t)cp2dst, */
1159: copy_syms_6a6:
1160: movzwl (%esi),%edx
1161: mov %dx,(%edi)
1162: add $0x2,%edi
1163: add $0x2,%esi
1164: sub $0x2,%eax
1165: jmp copy_syms_5dc
1166: copy_syms_6ba:
1167: mov -0x14(%ebp),%eax
1168: mov %eax,-0x10(%ebp)
1169: mov -0x20(%ebp),%eax
1170: mov %eax,-0x18(%ebp)
1171: /* cp1dst = (locore_Elf_Addr)(uintptr_t)endp; */
1172: mov 0x18(%ebp),%eax
1173: mov %eax,-0x24(%ebp)
1174: jmp copy_syms_486
1175: /* if (symtabp == NULL || strtabp == NULL) */
1176: copy_syms_6d1:
1177: test %eax,%eax
1178: jne copy_syms_3cb
1179: jmp copy_syms_4ce
1180: elf_sections_done:
1181: #endif
1182:
1183: jmp .Lbegin
1184:
1185:
1186: #endif /* MULTIBOOT */
1187:
1188: .Lnative_loader:
1.1 fvdl 1189: /*
1.84 maxv 1190: * Load parameters from the stack (32 bits):
1.91 maxv 1191: * boothowto, [bootdev], bootinfo, esym, biosextmem, biosbasemem
1.81 maxv 1192: * We are not interested in 'bootdev'.
1.1 fvdl 1193: */
1.81 maxv 1194:
1195: /* Load 'boothowto' */
1.1 fvdl 1196: movl 4(%esp),%eax
1197: movl %eax,RELOC(boothowto)
1.81 maxv 1198:
1199: /* Load 'bootinfo' */
1.1 fvdl 1200: movl 12(%esp),%eax
1.81 maxv 1201: testl %eax,%eax /* bootinfo = NULL? */
1.142 maxv 1202: jz .Lbootinfo_finished
1.81 maxv 1203:
1.121 maxv 1204: movl (%eax),%ebx /* bootinfo::bi_nentries */
1.1 fvdl 1205: movl $RELOC(bootinfo),%ebp
1.81 maxv 1206: movl %ebp,%edx
1.1 fvdl 1207: addl $BOOTINFO_MAXSIZE,%ebp
1.81 maxv 1208: movl %ebx,(%edx)
1209: addl $4,%edx
1210:
1.142 maxv 1211: .Lbootinfo_entryloop:
1.81 maxv 1212: testl %ebx,%ebx /* no remaining entries? */
1.142 maxv 1213: jz .Lbootinfo_finished
1.81 maxv 1214:
1215: addl $4,%eax
1216: movl (%eax),%ecx /* address of entry */
1.1 fvdl 1217: pushl %edi
1218: pushl %esi
1219: pushl %eax
1220:
1.84 maxv 1221: movl (%ecx),%eax /* btinfo_common::len (size of entry) */
1.1 fvdl 1222: movl %edx,%edi
1.121 maxv 1223: addl %eax,%edx /* update dest pointer */
1.81 maxv 1224: cmpl %ebp,%edx /* beyond bootinfo+BOOTINFO_MAXSIZE? */
1.142 maxv 1225: jg .Lbootinfo_overflow
1.81 maxv 1226:
1.1 fvdl 1227: movl %ecx,%esi
1228: movl %eax,%ecx
1.81 maxv 1229:
1.43 ad 1230: /*
1.121 maxv 1231: * If any modules were loaded, record where they end. 'eblob' is used
1232: * later to compute the initial bootstrap tables.
1.43 ad 1233: */
1.84 maxv 1234: cmpl $BTINFO_MODULELIST,4(%esi) /* btinfo_common::type */
1.142 maxv 1235: jne .Lbootinfo_copy
1.81 maxv 1236:
1.121 maxv 1237: /* Skip the modules if we won't have enough VA to map them */
1238: movl 12(%esi),%eax /* btinfo_modulelist::endpa */
1239: addl $PGOFSET,%eax /* roundup to a page */
1240: andl $~PGOFSET,%eax
1241: cmpl $BOOTMAP_VA_SIZE,%eax
1.142 maxv 1242: jg .Lbootinfo_skip
1.121 maxv 1243: movl %eax,RELOC(eblob)
1.81 maxv 1244: addl $KERNBASE_LO,RELOC(eblob)
1245: adcl $KERNBASE_HI,RELOC(eblob)+4
1246:
1.142 maxv 1247: .Lbootinfo_copy:
1.1 fvdl 1248: rep
1.81 maxv 1249: movsb /* copy esi -> edi */
1.142 maxv 1250: jmp .Lbootinfo_next
1.121 maxv 1251:
1.142 maxv 1252: .Lbootinfo_skip:
1.121 maxv 1253: subl %ecx,%edx /* revert dest pointer */
1254:
1.142 maxv 1255: .Lbootinfo_next:
1.1 fvdl 1256: popl %eax
1257: popl %esi
1258: popl %edi
1.81 maxv 1259: subl $1,%ebx /* decrement the # of entries */
1.142 maxv 1260: jmp .Lbootinfo_entryloop
1.81 maxv 1261:
1.142 maxv 1262: .Lbootinfo_overflow:
1.81 maxv 1263: /*
1264: * Cleanup for overflow case. Pop the registers, and correct the number
1265: * of entries.
1266: */
1.1 fvdl 1267: popl %eax
1268: popl %esi
1269: popl %edi
1270: movl $RELOC(bootinfo),%ebp
1.81 maxv 1271: movl %ebp,%edx
1272: subl %ebx,(%edx) /* correct the number of entries */
1.142 maxv 1273: .Lbootinfo_finished:
1.1 fvdl 1274:
1.81 maxv 1275: /* Load 'esym' */
1.72 uebayasi 1276: movl 16(%esp),%eax
1.81 maxv 1277: testl %eax,%eax /* esym = NULL? */
1.1 fvdl 1278: jz 1f
1.81 maxv 1279:
1.1 fvdl 1280: addl $KERNBASE_LO,%eax
1.81 maxv 1281:
1282: 1:
1283: movl $RELOC(esym),%ebp
1.1 fvdl 1284: movl %eax,(%ebp)
1285: movl $KERNBASE_HI,4(%ebp)
1286:
1.83 maxv 1287: /* Load 'biosextmem' */
1.1 fvdl 1288: movl $RELOC(biosextmem),%ebp
1289: movl (%ebp),%eax
1.83 maxv 1290: testl %eax,%eax /* already set? */
1.142 maxv 1291: jnz .Lbiosextmem_finished
1.81 maxv 1292:
1.1 fvdl 1293: movl 20(%esp),%eax
1294: movl %eax,(%ebp)
1.81 maxv 1295:
1.142 maxv 1296: .Lbiosextmem_finished:
1.83 maxv 1297: /* Load 'biosbasemem' */
1.1 fvdl 1298: movl $RELOC(biosbasemem),%ebp
1299: movl (%ebp),%eax
1.83 maxv 1300: testl %eax,%eax /* already set? */
1.142 maxv 1301: jnz .Lbiosbasemem_finished
1.83 maxv 1302:
1.1 fvdl 1303: movl 24(%esp),%eax
1304: movl %eax,(%ebp)
1305:
1.142 maxv 1306: .Lbiosbasemem_finished:
1.83 maxv 1307: /*
1.84 maxv 1308: * Done with the parameters!
1.83 maxv 1309: */
1.84 maxv 1310:
1.193 manu 1311: .Lbegin:
1.84 maxv 1312: /* First, reset the PSL. */
1.1 fvdl 1313: pushl $PSL_MBO
1314: popfl
1315:
1316: xorl %eax,%eax
1317: cpuid
1318: movl %eax,RELOC(cpuid_level)
1319:
1320: /*
1321: * Finished with old stack; load new %esp now instead of later so we
1322: * can trace this code without having to worry about the trace trap
1323: * clobbering the memory test or the zeroing of the bss+bootstrap page
1324: * tables.
1325: *
1326: * The boot program should check:
1327: * text+data <= &stack_variable - more_space_for_stack
1328: * text+data+bss+pad+space_for_page_tables <= end_of_memory
1.100 maxv 1329: *
1.84 maxv 1330: * XXX: the gdt is in the carcass of the boot program so clearing
1.1 fvdl 1331: * the rest of memory is still not possible.
1332: */
1333: movl $RELOC(tmpstk),%esp
1334:
1.86 maxv 1335: /*
1.180 maxv 1336: * Retrieve the NX/XD flag. We use the 32bit version of PTE_NX.
1.86 maxv 1337: */
1338: movl $0x80000001,%eax
1339: cpuid
1340: andl $CPUID_NOX,%edx
1.142 maxv 1341: jz .Lno_NOX
1.180 maxv 1342: movl $PTE_NX32,RELOC(nox_flag)
1.142 maxv 1343: .Lno_NOX:
1.86 maxv 1344:
1.1 fvdl 1345: /*
1.84 maxv 1346: * There are four levels of pages in amd64: PML4 -> PDP -> PD -> PT. They will
1347: * be referred to as: L4 -> L3 -> L2 -> L1.
1348: *
1349: * Virtual address space of the kernel:
1.97 maxv 1350: * +------+--------+------+-----+--------+---------------------+----------
1351: * | TEXT | RODATA | DATA | BSS | [SYMS] | [PRELOADED MODULES] | L4 ->
1352: * +------+--------+------+-----+--------+---------------------+----------
1353: * (1) (2) (3)
1354: *
1.101 maxv 1355: * --------------+-----+-----+----+-------------+
1356: * -> PROC0 STK -> L3 -> L2 -> L1 | ISA I/O MEM |
1357: * --------------+-----+-----+----+-------------+
1.97 maxv 1358: * (4)
1.84 maxv 1359: *
1.97 maxv 1360: * PROC0 STK is obviously not linked as a page level. It just happens to be
1361: * caught between L4 and L3.
1362: *
1363: * (PROC0 STK + L4 + L3 + L2 + L1) is later referred to as BOOTSTRAP TABLES.
1.1 fvdl 1364: *
1.116 maxv 1365: * ISA I/O MEM has no physical page allocated here, just virtual addresses.
1366: *
1.88 maxv 1367: * Important note: the kernel segments are properly 4k-aligned
1368: * (see kern.ldscript), so there's no need to enforce alignment.
1.1 fvdl 1369: */
1370:
1.84 maxv 1371: /* Find end of kernel image; brings us on (1). */
1.115 maxv 1372: movl $RELOC(__kernel_end),%edi
1.83 maxv 1373:
1.78 uebayasi 1374: #if (NKSYMS || defined(DDB) || defined(MODULAR)) && !defined(makeoptions_COPY_SYMTAB)
1.84 maxv 1375: /* Save the symbols (if loaded); brinds us on (2). */
1.1 fvdl 1376: movl RELOC(esym),%eax
1377: testl %eax,%eax
1378: jz 1f
1379: subl $KERNBASE_LO,%eax /* XXX */
1380: movl %eax,%edi
1381: 1:
1382: #endif
1.84 maxv 1383: /* Skip over any modules/blobs; brings us on (3). */
1.43 ad 1384: movl RELOC(eblob),%eax
1385: testl %eax,%eax
1386: jz 1f
1387: subl $KERNBASE_LO,%eax /* XXX */
1388: movl %eax,%edi
1389: 1:
1.81 maxv 1390:
1.97 maxv 1391: /* We are on (3). Align up for BOOTSTRAP TABLES. */
1.1 fvdl 1392: movl %edi,%esi
1.84 maxv 1393: addl $PGOFSET,%esi
1.1 fvdl 1394: andl $~PGOFSET,%esi
1395:
1.93 maxv 1396: /* We are on the BOOTSTRAP TABLES. Save L4's physical address. */
1397: movl $RELOC(PDPpaddr),%ebp
1398: movl %esi,(%ebp)
1399: movl $0,4(%ebp)
1400:
1.84 maxv 1401: /* Now, zero out the BOOTSTRAP TABLES (before filling them in). */
1.91 maxv 1402: movl %esi,%edi
1.1 fvdl 1403: xorl %eax,%eax
1404: cld
1405: movl $TABLESIZE,%ecx
1406: shrl $2,%ecx
1407: rep
1.83 maxv 1408: stosl /* copy eax -> edi */
1.1 fvdl 1409:
1.73 uebayasi 1410: /*
1.84 maxv 1411: * Build the page tables and levels. We go from L1 to L4, and link the levels
1412: * together. Note: RELOC computes &addr - KERNBASE in 32 bits; the value can't
1413: * be > 4G, or we can't deal with it anyway, since we are in 32bit mode.
1.73 uebayasi 1414: */
1.84 maxv 1415: /*
1416: * Build L1.
1417: */
1.83 maxv 1418: leal (PROC0_PTP1_OFF)(%esi),%ebx
1.81 maxv 1419:
1.119 maxv 1420: /* Skip the area below the kernel text. */
1.96 maxv 1421: movl $(KERNTEXTOFF_LO - KERNBASE_LO),%ecx
1422: shrl $PGSHIFT,%ecx
1423: fillkpt_blank
1.81 maxv 1424:
1.88 maxv 1425: /* Map the kernel text RX. */
1.96 maxv 1426: movl $(KERNTEXTOFF_LO - KERNBASE_LO),%eax /* start of TEXT */
1.88 maxv 1427: movl $RELOC(__rodata_start),%ecx
1428: subl %eax,%ecx
1429: shrl $PGSHIFT,%ecx
1.180 maxv 1430: orl $(PTE_P),%eax
1.88 maxv 1431: fillkpt
1.1 fvdl 1432:
1.91 maxv 1433: /* Map the kernel rodata R. */
1.88 maxv 1434: movl $RELOC(__rodata_start),%eax
1435: movl $RELOC(__data_start),%ecx
1.1 fvdl 1436: subl %eax,%ecx
1437: shrl $PGSHIFT,%ecx
1.180 maxv 1438: orl $(PTE_P),%eax
1.89 maxv 1439: fillkpt_nox
1.1 fvdl 1440:
1.90 maxv 1441: /* Map the kernel data+bss RW. */
1442: movl $RELOC(__data_start),%eax
1443: movl $RELOC(__kernel_end),%ecx
1444: subl %eax,%ecx
1445: shrl $PGSHIFT,%ecx
1.180 maxv 1446: orl $(PTE_P|PTE_W),%eax
1.90 maxv 1447: fillkpt_nox
1448:
1.105 maxv 1449: /* Map [SYMS]+[PRELOADED MODULES] RW. */
1.90 maxv 1450: movl $RELOC(__kernel_end),%eax
1.97 maxv 1451: movl %esi,%ecx /* start of BOOTSTRAP TABLES */
1.95 maxv 1452: subl %eax,%ecx
1453: shrl $PGSHIFT,%ecx
1.180 maxv 1454: orl $(PTE_P|PTE_W),%eax
1.105 maxv 1455: fillkpt_nox
1.95 maxv 1456:
1.92 maxv 1457: /* Map the BOOTSTRAP TABLES RW. */
1458: movl %esi,%eax /* start of BOOTSTRAP TABLES */
1459: movl $TABLESIZE,%ecx /* length of BOOTSTRAP TABLES */
1460: shrl $PGSHIFT,%ecx
1.180 maxv 1461: orl $(PTE_P|PTE_W),%eax
1.92 maxv 1462: fillkpt_nox
1463:
1.102 maxv 1464: /* We are on (4). Map ISA I/O MEM RW. */
1.101 maxv 1465: movl $IOM_BEGIN,%eax
1466: movl $IOM_SIZE,%ecx /* size of ISA I/O MEM */
1467: shrl $PGSHIFT,%ecx
1.180 maxv 1468: orl $(PTE_P|PTE_W/*|PTE_PCD*/),%eax
1.102 maxv 1469: fillkpt_nox
1.1 fvdl 1470:
1.84 maxv 1471: /*
1472: * Build L2. Linked to L1.
1473: */
1.73 uebayasi 1474: leal (PROC0_PTP2_OFF)(%esi),%ebx
1.1 fvdl 1475: leal (PROC0_PTP1_OFF)(%esi),%eax
1.180 maxv 1476: orl $(PTE_P|PTE_W),%eax
1.1 fvdl 1477: movl $(NKL2_KIMG_ENTRIES+1),%ecx
1478: fillkpt
1479:
1480: #if L2_SLOT_KERNBASE > 0
1481: /* If needed, set up level 2 entries for actual kernel mapping */
1.84 maxv 1482: leal (PROC0_PTP2_OFF + L2_SLOT_KERNBASE * PDE_SIZE)(%esi),%ebx
1.73 uebayasi 1483: leal (PROC0_PTP1_OFF)(%esi),%eax
1.180 maxv 1484: orl $(PTE_P|PTE_W),%eax
1.73 uebayasi 1485: movl $(NKL2_KIMG_ENTRIES+1),%ecx
1.1 fvdl 1486: fillkpt
1487: #endif
1488:
1.84 maxv 1489: /*
1490: * Build L3. Linked to L2.
1491: */
1.73 uebayasi 1492: leal (PROC0_PTP3_OFF)(%esi),%ebx
1.1 fvdl 1493: leal (PROC0_PTP2_OFF)(%esi),%eax
1.180 maxv 1494: orl $(PTE_P|PTE_W),%eax
1.1 fvdl 1495: movl $NKL3_KIMG_ENTRIES,%ecx
1496: fillkpt
1497:
1498: #if L3_SLOT_KERNBASE > 0
1499: /* If needed, set up level 3 entries for actual kernel mapping */
1.84 maxv 1500: leal (PROC0_PTP3_OFF + L3_SLOT_KERNBASE * PDE_SIZE)(%esi),%ebx
1.73 uebayasi 1501: leal (PROC0_PTP2_OFF)(%esi),%eax
1.180 maxv 1502: orl $(PTE_P|PTE_W),%eax
1.73 uebayasi 1503: movl $NKL3_KIMG_ENTRIES,%ecx
1.1 fvdl 1504: fillkpt
1505: #endif
1506:
1.84 maxv 1507: /*
1508: * Build L4 for identity mapping. Linked to L3.
1509: */
1.73 uebayasi 1510: leal (PROC0_PML4_OFF)(%esi),%ebx
1.1 fvdl 1511: leal (PROC0_PTP3_OFF)(%esi),%eax
1.180 maxv 1512: orl $(PTE_P|PTE_W),%eax
1.1 fvdl 1513: movl $NKL4_KIMG_ENTRIES,%ecx
1514: fillkpt
1515:
1.84 maxv 1516: /* Set up L4 entries for actual kernel mapping */
1517: leal (PROC0_PML4_OFF + L4_SLOT_KERNBASE * PDE_SIZE)(%esi),%ebx
1.1 fvdl 1518: leal (PROC0_PTP3_OFF)(%esi),%eax
1.180 maxv 1519: orl $(PTE_P|PTE_W),%eax
1.1 fvdl 1520: movl $NKL4_KIMG_ENTRIES,%ecx
1521: fillkpt
1522:
1523: /*
1524: * Startup checklist:
1525: * 1. Enable PAE (and SSE while here).
1526: */
1527: movl %cr4,%eax
1528: orl $(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax
1529: movl %eax,%cr4
1530:
1531: /*
1.86 maxv 1532: * 2. Set Long Mode Enable in EFER. Also enable the syscall extensions,
1533: * and NOX if available.
1.1 fvdl 1534: */
1.73 uebayasi 1535: movl $MSR_EFER,%ecx
1.1 fvdl 1536: rdmsr
1537: xorl %eax,%eax /* XXX */
1538: orl $(EFER_LME|EFER_SCE),%eax
1.86 maxv 1539: movl RELOC(nox_flag),%ebx
1540: cmpl $0,%ebx
1.142 maxv 1541: je .Lskip_NOX
1.86 maxv 1542: orl $(EFER_NXE),%eax
1.142 maxv 1543: .Lskip_NOX:
1.1 fvdl 1544: wrmsr
1545:
1546: /*
1547: * 3. Load %cr3 with pointer to PML4.
1548: */
1549: movl %esi,%eax
1550: movl %eax,%cr3
1551:
1552: /*
1553: * 4. Enable paging and the rest of it.
1554: */
1555: movl %cr0,%eax
1.68 jym 1556: orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax
1.1 fvdl 1557: movl %eax,%cr0
1558: jmp compat
1559: compat:
1560:
1561: /*
1.83 maxv 1562: * 5. Not quite done yet, we're now in a compatibility segment, in
1563: * legacy mode. We must jump to a long mode segment. Need to set up
1564: * a temporary GDT with a long mode segment in it to do that.
1.1 fvdl 1565: */
1.56 jym 1566: movl $RELOC(gdt64_lo),%eax
1.1 fvdl 1567: lgdt (%eax)
1568: movl $RELOC(farjmp64),%eax
1569: ljmp *(%eax)
1570:
1.83 maxv 1571: .code64
1.1 fvdl 1572: longmode:
1573: /*
1.83 maxv 1574: * 6. Finally, we're in long mode. However, we're still in the identity
1575: * mapped area (could not jump out of that earlier because it would
1576: * have been a > 32bit jump). We can do that now, so here we go.
1.1 fvdl 1577: */
1578: movabsq $longmode_hi,%rax
1579: jmp *%rax
1.56 jym 1580:
1.1 fvdl 1581: longmode_hi:
1.56 jym 1582:
1583: /*
1584: * We left the identity mapped area. Base address of
1585: * the temporary gdt64 should now be in high memory.
1586: */
1587: movq $RELOC(gdt64_hi),%rax
1588: lgdt (%rax)
1589:
1.1 fvdl 1590: /*
1.83 maxv 1591: * We have arrived. There's no need anymore for the identity mapping in
1592: * low memory, remove it.
1.1 fvdl 1593: */
1594: movq $KERNBASE,%r8
1595:
1596: #if L2_SLOT_KERNBASE > 0
1597: movq $(NKL2_KIMG_ENTRIES+1),%rcx
1.84 maxv 1598: leaq (PROC0_PTP2_OFF)(%rsi),%rbx /* old, phys address */
1599: addq %r8,%rbx /* new, virt address */
1600: killkpt
1.1 fvdl 1601: #endif
1602:
1603: #if L3_SLOT_KERNBASE > 0
1604: movq $NKL3_KIMG_ENTRIES,%rcx
1.84 maxv 1605: leaq (PROC0_PTP3_OFF)(%rsi),%rbx /* old, phys address */
1606: addq %r8,%rbx /* new, virt address */
1607: killkpt
1.1 fvdl 1608: #endif
1609:
1610: movq $NKL4_KIMG_ENTRIES,%rcx
1.84 maxv 1611: leaq (PROC0_PML4_OFF)(%rsi),%rbx /* old, phys address of PML4 */
1612: addq %r8,%rbx /* new, virt address of PML4 */
1.85 maxv 1613: killkpt
1.1 fvdl 1614:
1615: /* Relocate atdevbase. */
1616: movq $(TABLESIZE+KERNBASE),%rdx
1617: addq %rsi,%rdx
1618: movq %rdx,_C_LABEL(atdevbase)(%rip)
1619:
1620: /* Set up bootstrap stack. */
1.97 maxv 1621: leaq (PROC0_STK_OFF)(%rsi),%rax
1.1 fvdl 1622: addq %r8,%rax
1.130 maxv 1623: movq %rax,_C_LABEL(lwp0uarea)(%rip)
1.1 fvdl 1624: leaq (USPACE-FRAMESIZE)(%rax),%rsp
1.84 maxv 1625: xorq %rbp,%rbp /* mark end of frames */
1.1 fvdl 1626:
1.193 manu 1627: #if defined(MULTIBOOT)
1628: /* It is now safe to parse the Multiboot information structure
1629: * we saved before from C code. Note that we cannot delay its
1630: * parsing any more because initgdt (called below) needs to make
1631: * use of this information.
1632: */
1633: pushq %rsi
1634: call _C_LABEL(multiboot2_post_reloc)
1635: popq %rsi
1636: #endif
1.1 fvdl 1637: xorw %ax,%ax
1638: movw %ax,%gs
1639: movw %ax,%fs
1640:
1.116 maxv 1641: /* The first physical page available. */
1642: leaq (TABLESIZE)(%rsi),%rdi
1.32 bouyer 1643:
1.176 cherry 1644: #else /* XENPV */
1.32 bouyer 1645: /* First, reset the PSL. */
1646: pushq $2
1647: popfq
1648:
1649: cld
1650:
1651: /*
1652: * Xen info:
1653: * - %rsi -> start_info struct
1.111 maxv 1654: * - %rsp -> stack, *theoretically* the last used page by Xen bootstrap
1.32 bouyer 1655: */
1.111 maxv 1656: movq %rsi,%rbx
1.32 bouyer 1657:
1.73 uebayasi 1658: /* Clear BSS. */
1.32 bouyer 1659: xorq %rax,%rax
1660: movq $_C_LABEL(__bss_start),%rdi
1661: movq $_C_LABEL(_end),%rcx
1662: subq %rdi,%rcx
1663: rep
1664: stosb
1665:
1.111 maxv 1666: /* Copy start_info to a safe place. */
1.32 bouyer 1667: movq %rbx,%rsi
1668: movq $_C_LABEL(start_info_union),%rdi
1669: movq $64,%rcx
1670: rep
1671: movsq
1672:
1673: /*
1674: * Memory layout at start of the day:
1675: * - Kernel image
1676: * - Page frames list
1677: * - start_info struct. we copied it, so it can be recycled.
1678: * - xenstore
1679: * - console
1680: * - Xen bootstrap page tables
1681: * - kernel stack. provided by Xen
1.79 maxv 1682: * - guaranteed 512kB padding
1.32 bouyer 1683: *
1684: * As we want to rebuild our page tables and place our stack
1685: * in proc0 struct, all data starting from after console can be
1686: * discarded after we've done a little setup.
1687: */
1688:
1689: /*
1.111 maxv 1690: * We want our own page tables, and will rebuild them. We will reclaim
1691: * the Xen space later, INCLUDING the stack. So we need to switch to a
1692: * temporary one now.
1693: */
1694: movq $tmpstk,%rax
1695: subq $8,%rax
1696: movq %rax,%rsp
1.32 bouyer 1697:
1.45 bouyer 1698: xorl %eax,%eax
1699: cpuid
1700: movl %eax,_C_LABEL(cpuid_level)
1701:
1.111 maxv 1702: movq $cpu_info_primary,%rdi
1703: movq %rdi,CPU_INFO_SELF(%rdi) /* ci->ci_self = ci */
1704: movq $1,%rsi
1.67 cherry 1705: call cpu_init_msrs /* cpu_init_msrs(ci, true); */
1706:
1.113 maxv 1707: call xen_locore
1.32 bouyer 1708:
1709: /*
1.113 maxv 1710: * The first VA available is returned by xen_locore in %rax. We
1.111 maxv 1711: * use it as the UAREA, and set up the stack here.
1.32 bouyer 1712: */
1.111 maxv 1713: movq %rax,%rsi
1.130 maxv 1714: movq %rsi,_C_LABEL(lwp0uarea)(%rip)
1.32 bouyer 1715: leaq (USPACE-FRAMESIZE)(%rsi),%rsp
1716: xorq %rbp,%rbp
1717:
1.111 maxv 1718: /* Clear segment registers. */
1.63 jym 1719: xorw %ax,%ax
1.32 bouyer 1720: movw %ax,%gs
1721: movw %ax,%fs
1.84 maxv 1722:
1.117 maxv 1723: /* Set first_avail after the DUMMY PAGE (see xen_locore). */
1.32 bouyer 1724: movq %rsi,%rdi
1.117 maxv 1725: addq $(USPACE+PAGE_SIZE),%rdi
1.84 maxv 1726: subq $KERNBASE,%rdi /* init_x86_64 wants a physical address */
1.176 cherry 1727: #endif /* XENPV */
1.32 bouyer 1728:
1.131 maxv 1729: pushq %rdi
1730: call _C_LABEL(init_bootspace)
1.172 maxv 1731: call _C_LABEL(init_slotspace)
1.131 maxv 1732: popq %rdi
1.1 fvdl 1733: call _C_LABEL(init_x86_64)
1734: call _C_LABEL(main)
1.71 uebayasi 1735: END(start)
1.1 fvdl 1736:
1.73 uebayasi 1737: #if defined(XEN)
1.32 bouyer 1738: /* space for the hypercall call page */
1739: #define HYPERCALL_PAGE_OFFSET 0x1000
1.177 cherry 1740: .align HYPERCALL_PAGE_OFFSET
1741: ENTRY(hypercall_page) /* Returns -1, on HYPERVISOR_xen_version() */
1742: .skip (__HYPERVISOR_xen_version*32), 0x90
1743: movq $-1, %rax
1744: retq
1745: .align HYPERCALL_PAGE_OFFSET, 0x90
1.71 uebayasi 1746: END(hypercall_page)
1.32 bouyer 1747: #endif /* XEN */
1.1 fvdl 1748:
1749: /*
1.22 yamt 1750: * int setjmp(label_t *)
1751: *
1752: * Used primarily by DDB.
1753: */
1.1 fvdl 1754: ENTRY(setjmp)
1755: /*
1756: * Only save registers that must be preserved across function
1757: * calls according to the ABI (%rbx, %rsp, %rbp, %r12-%r15)
1758: * and %rip.
1759: */
1760: movq %rdi,%rax
1761: movq %rbx,(%rax)
1762: movq %rsp,8(%rax)
1763: movq %rbp,16(%rax)
1764: movq %r12,24(%rax)
1765: movq %r13,32(%rax)
1766: movq %r14,40(%rax)
1767: movq %r15,48(%rax)
1768: movq (%rsp),%rdx
1769: movq %rdx,56(%rax)
1770: xorl %eax,%eax
1771: ret
1.71 uebayasi 1772: END(setjmp)
1.1 fvdl 1773:
1.22 yamt 1774: /*
1775: * int longjmp(label_t *)
1776: *
1777: * Used primarily by DDB.
1778: */
1.1 fvdl 1779: ENTRY(longjmp)
1780: movq %rdi,%rax
1781: movq (%rax),%rbx
1782: movq 8(%rax),%rsp
1783: movq 16(%rax),%rbp
1784: movq 24(%rax),%r12
1785: movq 32(%rax),%r13
1786: movq 40(%rax),%r14
1787: movq 48(%rax),%r15
1788: movq 56(%rax),%rdx
1789: movq %rdx,(%rsp)
1.22 yamt 1790: movl $1,%eax
1.1 fvdl 1791: ret
1.71 uebayasi 1792: END(longjmp)
1.1 fvdl 1793:
1.73 uebayasi 1794: /*
1795: * void dumpsys(void)
1796: *
1797: * Mimic cpu_switchto() for postmortem debugging.
1798: */
1.25 yamt 1799: ENTRY(dumpsys)
1.84 maxv 1800: /* Build a fake switch frame. */
1.25 yamt 1801: pushq %rbx
1802: pushq %r12
1803: pushq %r13
1804: pushq %r14
1805: pushq %r15
1.84 maxv 1806:
1807: /* Save a context. */
1.25 yamt 1808: movq $dumppcb, %rax
1809: movq %rsp, PCB_RSP(%rax)
1810: movq %rbp, PCB_RBP(%rax)
1811:
1812: call _C_LABEL(dodumpsys)
1813:
1.84 maxv 1814: addq $(5*8), %rsp /* sizeof(switchframe) - sizeof(%rip) */
1.25 yamt 1815: ret
1.71 uebayasi 1816: END(dumpsys)
1.25 yamt 1817:
1.1 fvdl 1818: /*
1.58 chs 1819: * struct lwp *cpu_switchto(struct lwp *oldlwp, struct lwp *newlwp,
1.103 maxv 1820: * bool returning)
1.22 yamt 1821: *
1822: * 1. if (oldlwp != NULL), save its context.
1823: * 2. then, restore context of newlwp.
1824: *
1825: * Note that the stack frame layout is known to "struct switchframe" in
1826: * <machine/frame.h> and to the code in cpu_lwp_fork() which initializes
1827: * it for a new lwp.
1.1 fvdl 1828: */
1.22 yamt 1829: ENTRY(cpu_switchto)
1.1 fvdl 1830: pushq %rbx
1831: pushq %r12
1832: pushq %r13
1833: pushq %r14
1834: pushq %r15
1835:
1.84 maxv 1836: movq %rdi,%r13 /* oldlwp */
1837: movq %rsi,%r12 /* newlwp */
1.1 fvdl 1838:
1.22 yamt 1839: /* Save old context. */
1.55 rmind 1840: movq L_PCB(%r13),%rax
1.22 yamt 1841: movq %rsp,PCB_RSP(%rax)
1842: movq %rbp,PCB_RBP(%rax)
1843:
1.168 maxv 1844: /* Switch to newlwp's stack. */
1845: movq L_PCB(%r12),%r14
1846: movq PCB_RSP(%r14),%rsp
1847: movq PCB_RBP(%r14),%rbp
1848:
1849: /*
1850: * Set curlwp. This must be globally visible in order to permit
1851: * non-interlocked mutex release.
1852: */
1853: movq %r12,%rcx
1854: xchgq %rcx,CPUVAR(CURLWP)
1855:
1856: /* Skip the rest if returning to a pinned LWP. */
1857: testb %dl,%dl /* returning = true ? */
1858: jnz .Lswitch_return
1859:
1.145 maxv 1860: #ifdef SVS
1.151 maxv 1861: movb _C_LABEL(svs_enabled),%dl
1862: testb %dl,%dl
1863: jz .Lskip_svs
1.145 maxv 1864: callq _C_LABEL(svs_lwp_switch)
1.151 maxv 1865: .Lskip_svs:
1.145 maxv 1866: #endif
1867:
1.178 cherry 1868: #ifndef XEN
1.168 maxv 1869: movq %r13,%rdi
1870: movq %r12,%rsi
1.160 maxv 1871: callq _C_LABEL(speculation_barrier)
1.163 maxv 1872: #endif
1.160 maxv 1873:
1.38 yamt 1874: /* Switch ring0 stack */
1.147 maxv 1875: #ifdef SVS
1.152 maxv 1876: movb _C_LABEL(svs_enabled),%al
1877: testb %al,%al
1878: jz .Lno_svs_switch
1879:
1.147 maxv 1880: movq CPUVAR(RSP0),%rax
1881: movq CPUVAR(TSS),%rdi
1882: movq %rax,TSS_RSP0(%rdi)
1.152 maxv 1883: jmp .Lring0_switched
1884:
1885: .Lno_svs_switch:
1886: #endif
1887:
1.176 cherry 1888: #if !defined(XENPV)
1.38 yamt 1889: movq PCB_RSP0(%r14),%rax
1.144 maxv 1890: movq CPUVAR(TSS),%rdi
1891: movq %rax,TSS_RSP0(%rdi)
1.38 yamt 1892: #else
1.103 maxv 1893: movq %r14,%rdi
1.152 maxv 1894: callq _C_LABEL(x86_64_switch_context)
1.32 bouyer 1895: #endif
1.152 maxv 1896: .Lring0_switched:
1.1 fvdl 1897:
1.173 maxv 1898: /* Switch the dbregs. */
1899: movq %r13,%rdi
1900: movq %r12,%rsi
1901: callq _C_LABEL(x86_dbregs_switch)
1902:
1.187 maxv 1903: /* Switch the FPU. */
1.167 maxv 1904: movq %r13,%rdi
1905: movq %r12,%rsi
1.188 maxv 1906: callq _C_LABEL(fpu_switch)
1.167 maxv 1907:
1.22 yamt 1908: /* Don't bother with the rest if switching to a system process. */
1909: testl $LW_SYSTEM,L_FLAG(%r12)
1.142 maxv 1910: jnz .Lswitch_return
1.1 fvdl 1911:
1.22 yamt 1912: /* Is this process using RAS (restartable atomic sequences)? */
1913: movq L_PROC(%r12),%rdi
1914: cmpq $0,P_RASLIST(%rdi)
1.142 maxv 1915: je .Lno_RAS
1.104 maxv 1916:
1917: /* Handle restartable atomic sequences (RAS). */
1918: movq L_MD_REGS(%r12),%rbx
1919: movq TF_RIP(%rbx),%rsi
1920: call _C_LABEL(ras_lookup)
1921: cmpq $-1,%rax
1.142 maxv 1922: je .Lno_RAS
1.104 maxv 1923: movq %rax,TF_RIP(%rbx)
1.142 maxv 1924: .Lno_RAS:
1.1 fvdl 1925:
1.176 cherry 1926: #ifndef XENPV
1.189 maxv 1927: /* Raise the IPL to IPL_HIGH. Dropping the priority is deferred until
1928: * mi_switch(), when cpu_switchto() returns. XXX Still needed? */
1.75 christos 1929: movl $IPL_HIGH,CPUVAR(ILEVEL)
1.58 chs 1930:
1.103 maxv 1931: /* The 32bit LWPs are handled differently. */
1932: testl $PCB_COMPAT32,PCB_FLAGS(%r14)
1.142 maxv 1933: jnz .Llwp_32bit
1.58 chs 1934:
1.142 maxv 1935: .Llwp_64bit:
1.134 maxv 1936: /* Set default 64bit values in %ds, %es, %fs and %gs. */
1937: movq $GSEL(GUDATA_SEL, SEL_UPL),%rax
1938: movw %ax,%ds
1939: movw %ax,%es
1.103 maxv 1940: xorq %rax,%rax
1941: movw %ax,%fs
1.58 chs 1942: CLI(cx)
1.70 dsl 1943: SWAPGS
1.103 maxv 1944: movw %ax,%gs
1.70 dsl 1945: SWAPGS
1.58 chs 1946: STI(cx)
1947:
1.107 maxv 1948: /* Zero out GDT descriptors. */
1.58 chs 1949: movq CPUVAR(GDT),%rcx
1.103 maxv 1950: movq %rax,(GUFS_SEL*8)(%rcx)
1951: movq %rax,(GUGS_SEL*8)(%rcx)
1.58 chs 1952:
1953: /* Reload 64-bit %fs/%gs MSRs. */
1.103 maxv 1954: movl $MSR_FSBASE,%ecx
1955: movl PCB_FS(%r14),%eax
1956: movl 4+PCB_FS(%r14),%edx
1.58 chs 1957: wrmsr
1.103 maxv 1958: movl $MSR_KERNELGSBASE,%ecx
1959: movl PCB_GS(%r14),%eax
1960: movl 4+PCB_GS(%r14),%edx
1.58 chs 1961: wrmsr
1.107 maxv 1962:
1.142 maxv 1963: jmp .Lswitch_return
1.58 chs 1964:
1.142 maxv 1965: .Llwp_32bit:
1.58 chs 1966: /* Reload %fs/%gs GDT descriptors. */
1967: movq CPUVAR(GDT),%rcx
1.103 maxv 1968: movq PCB_FS(%r14),%rax
1969: movq %rax,(GUFS_SEL*8)(%rcx)
1970: movq PCB_GS(%r14),%rax
1971: movq %rax,(GUGS_SEL*8)(%rcx)
1.58 chs 1972:
1.143 maxv 1973: /* Set default 32bit values in %ds, %es, %fs and %gs. */
1.103 maxv 1974: movq L_MD_REGS(%r12),%rbx
1.134 maxv 1975: movq $GSEL(GUDATA32_SEL, SEL_UPL),%rax
1976: movw %ax,%ds
1977: movw %ax,%es
1.135 maxv 1978: movw %ax,%fs
1.58 chs 1979: CLI(ax)
1.70 dsl 1980: SWAPGS
1.137 maxv 1981: movw %ax,%gs
1.70 dsl 1982: SWAPGS
1.58 chs 1983: STI(ax)
1984: #else
1985: movq %r12,%rdi
1986: callq _C_LABEL(x86_64_tls_switch)
1.32 bouyer 1987: #endif
1.84 maxv 1988:
1.142 maxv 1989: .Lswitch_return:
1.22 yamt 1990: /* Return to the new LWP, returning 'oldlwp' in %rax. */
1.190 maxv 1991: KMSAN_INIT_RET(8)
1.103 maxv 1992: movq %r13,%rax
1.1 fvdl 1993: popq %r15
1994: popq %r14
1995: popq %r13
1996: popq %r12
1997: popq %rbx
1998: ret
1.71 uebayasi 1999: END(cpu_switchto)
1.1 fvdl 2000:
2001: /*
1.22 yamt 2002: * void savectx(struct pcb *pcb);
2003: *
1.1 fvdl 2004: * Update pcb, saving current processor state.
2005: */
2006: ENTRY(savectx)
2007: /* Save stack pointers. */
2008: movq %rsp,PCB_RSP(%rdi)
2009: movq %rbp,PCB_RBP(%rdi)
2010: ret
1.71 uebayasi 2011: END(savectx)
1.1 fvdl 2012:
2013: /*
1.148 maxv 2014: * Syscall handler.
1.1 fvdl 2015: */
1.171 maxv 2016: ENTRY(handle_syscall)
1.127 maxv 2017: STI(si)
1.1 fvdl 2018:
2019: movq CPUVAR(CURLWP),%r14
1.84 maxv 2020: incq CPUVAR(NSYSCALL) /* count it atomically */
2021: movq %rsp,L_MD_REGS(%r14) /* save pointer to frame */
1.1 fvdl 2022: movq L_PROC(%r14),%r15
1.70 dsl 2023: andl $~MDL_IRET,L_MD_FLAGS(%r14) /* Allow sysret return */
1.37 dsl 2024: movq %rsp,%rdi /* Pass frame as arg0 */
1.1 fvdl 2025: call *P_MD_SYSCALL(%r15)
1.29 yamt 2026: .Lsyscall_checkast:
1.70 dsl 2027: /*
2028: * Disable interrupts to avoid new ASTs (etc) being added and
2029: * to ensure we don't take an interrupt with some of the user
2030: * registers loaded.
2031: */
2032: CLI(si)
1.29 yamt 2033: /* Check for ASTs on exit to user mode. */
1.107 maxv 2034: movl L_MD_ASTPENDING(%r14),%eax
2035: orl CPUVAR(WANT_PMAPLOAD),%eax
1.29 yamt 2036: jnz 9f
1.107 maxv 2037:
1.1 fvdl 2038: #ifdef DIAGNOSTIC
2039: cmpl $IPL_NONE,CPUVAR(ILEVEL)
1.142 maxv 2040: jne .Lspl_error
1.1 fvdl 2041: #endif
1.107 maxv 2042:
1.189 maxv 2043: HANDLE_DEFERRED_FPU
2044:
1.129 maxv 2045: /*
1.134 maxv 2046: * Decide if we need to take a slow path. That's the case when we
2047: * want to reload %cs and %ss on a 64bit LWP (MDL_IRET set), or when
2048: * we're returning to a 32bit LWP (MDL_COMPAT32 set).
2049: *
2050: * In either case, we jump into intrfastexit and return to userland
2051: * with the iret instruction.
1.129 maxv 2052: */
1.107 maxv 2053: testl $(MDL_IRET|MDL_COMPAT32),L_MD_FLAGS(%r14)
1.134 maxv 2054: jnz intrfastexit
2055:
1.148 maxv 2056: jmp syscall_sysret
1.1 fvdl 2057:
2058: #ifdef DIAGNOSTIC
1.142 maxv 2059: .Lspl_error:
1.107 maxv 2060: movabsq $4f,%rdi
1.186 maxv 2061: movl CPUVAR(ILEVEL),%esi
2062: call _C_LABEL(panic)
2063: 4: .asciz "spl not lowered on syscall, ilevel=%x"
1.1 fvdl 2064: #endif
1.70 dsl 2065:
2066: /* AST pending or pmap load needed */
1.41 ad 2067: 9:
1.107 maxv 2068: cmpl $0,CPUVAR(WANT_PMAPLOAD)
1.41 ad 2069: jz 10f
2070: STI(si)
1.29 yamt 2071: call _C_LABEL(do_pmap_load)
2072: jmp .Lsyscall_checkast /* re-check ASTs */
1.41 ad 2073: 10:
2074: CLEAR_ASTPENDING(%r14)
2075: STI(si)
2076: /* Pushed T_ASTFLT into tf_trapno on entry. */
2077: movq %rsp,%rdi
1.190 maxv 2078: KMSAN_INIT_ARG(8)
1.41 ad 2079: call _C_LABEL(trap)
2080: jmp .Lsyscall_checkast /* re-check ASTs */
1.148 maxv 2081: END(handle_syscall)
1.29 yamt 2082:
1.22 yamt 2083: /*
2084: * void lwp_trampoline(void);
2085: *
2086: * This is a trampoline function pushed run by newly created LWPs
1.70 dsl 2087: * in order to do additional setup in their context.
1.22 yamt 2088: */
1.171 maxv 2089: ENTRY(lwp_trampoline)
1.22 yamt 2090: movq %rbp,%rsi
1.41 ad 2091: movq %rbp,%r14 /* for .Lsyscall_checkast */
1.22 yamt 2092: movq %rax,%rdi
2093: xorq %rbp,%rbp
1.190 maxv 2094: KMSAN_INIT_ARG(16)
1.22 yamt 2095: call _C_LABEL(lwp_startup)
1.1 fvdl 2096: movq %r13,%rdi
1.190 maxv 2097: KMSAN_INIT_ARG(8)
1.1 fvdl 2098: call *%r12
1.41 ad 2099: jmp .Lsyscall_checkast
1.71 uebayasi 2100: END(lwp_trampoline)
1.1 fvdl 2101:
2102: /*
1.148 maxv 2103: * Entry points of the 'syscall' instruction, 64bit and 32bit mode.
2104: */
2105:
1.156 maxv 2106: #define SP(x) (x)-(TF_SS+8)(%rax)
1.152 maxv 2107:
2108: .macro SYSCALL_ENTRY name,is_svs
2109: IDTVEC(\name)
1.176 cherry 2110: #ifndef XENPV
1.148 maxv 2111: /*
2112: * The user %rip is in %rcx and the user %rflags in %r11. The kernel %cs
2113: * and %ss are loaded, but nothing else is.
2114: *
2115: * The 'swapgs' instruction gives us access to cpu-specific memory where
2116: * we can save a user register and then read the LWP's kernel stack
2117: * pointer.
2118: *
2119: * This code doesn't seem to set %ds, this may not matter since it is
2120: * ignored in 64bit mode, OTOH the syscall instruction sets %ss and that
2121: * is ignored as well.
2122: */
2123: swapgs
2124:
1.156 maxv 2125: /* Get the LWP's kernel stack pointer in %rax */
1.152 maxv 2126: .if \is_svs
1.165 joerg 2127: movabs %rax,SVS_UTLS+UTLS_SCRATCH
2128: movabs SVS_UTLS+UTLS_RSP0,%rax
1.156 maxv 2129: .else
2130: movq %rax,CPUVAR(SCRATCH)
2131: movq CPUVAR(CURLWP),%rax
2132: movq L_PCB(%rax),%rax
2133: movq PCB_RSP0(%rax),%rax
2134: .endif
1.152 maxv 2135:
1.156 maxv 2136: /* Make stack look like an 'int nn' frame */
2137: movq $(LSEL(LUDATA_SEL, SEL_UPL)),SP(TF_SS) /* user %ss */
2138: movq %rsp,SP(TF_RSP) /* user %rsp */
2139: movq %r11,SP(TF_RFLAGS) /* user %rflags */
2140: movq $(LSEL(LUCODE_SEL, SEL_UPL)),SP(TF_CS) /* user %cs */
2141: movq %rcx,SP(TF_RIP) /* user %rip */
2142: leaq SP(0),%rsp /* %rsp now valid after frame */
1.152 maxv 2143:
1.156 maxv 2144: /* Restore %rax */
2145: .if \is_svs
1.165 joerg 2146: movabs SVS_UTLS+UTLS_SCRATCH,%rax
1.152 maxv 2147: .else
1.156 maxv 2148: movq CPUVAR(SCRATCH),%rax
1.152 maxv 2149: .endif
1.148 maxv 2150:
2151: movq $2,TF_ERR(%rsp) /* syscall instruction size */
2152: movq $T_ASTFLT,TF_TRAPNO(%rsp)
2153: #else
2154: /* Xen already switched to kernel stack */
2155: addq $0x10,%rsp /* gap to match cs:rip */
2156: pushq $2 /* error code */
2157: pushq $T_ASTFLT
2158: subq $TF_REGSIZE,%rsp
2159: cld
2160: #endif
2161: INTR_SAVE_GPRS
1.160 maxv 2162: IBRS_ENTER
1.148 maxv 2163: movw $GSEL(GUDATA_SEL, SEL_UPL),TF_DS(%rsp)
2164: movw $GSEL(GUDATA_SEL, SEL_UPL),TF_ES(%rsp)
2165: movw $0,TF_FS(%rsp)
2166: movw $0,TF_GS(%rsp)
1.152 maxv 2167: .if \is_svs
2168: SVS_ENTER
2169: .endif
1.190 maxv 2170: KMSAN_ENTER
1.148 maxv 2171: jmp handle_syscall
1.152 maxv 2172: IDTVEC_END(\name)
2173: .endm
2174:
2175: SYSCALL_ENTRY syscall,is_svs=0
2176:
2177: TEXT_USER_BEGIN
2178:
1.155 maxv 2179: #ifdef SVS
1.152 maxv 2180: SYSCALL_ENTRY syscall_svs,is_svs=1
1.155 maxv 2181: #endif
1.148 maxv 2182:
2183: IDTVEC(syscall32)
2184: sysret /* go away please */
2185: IDTVEC_END(syscall32)
2186:
2187: TEXT_USER_END
2188:
2189: /*
1.22 yamt 2190: * osyscall()
2191: *
1.1 fvdl 2192: * Trap gate entry for int $80 syscall, also used by sigreturn.
2193: */
1.148 maxv 2194: TEXT_USER_BEGIN
1.1 fvdl 2195: IDTVEC(osyscall)
1.176 cherry 2196: #ifdef XENPV
1.32 bouyer 2197: movq (%rsp),%rcx
2198: movq 8(%rsp),%r11
2199: addq $0x10,%rsp
2200: #endif
1.84 maxv 2201: pushq $2 /* size of instruction for restart */
2202: pushq $T_ASTFLT /* trap # for doing ASTs */
1.1 fvdl 2203: INTRENTRY
1.148 maxv 2204: jmp handle_syscall
1.71 uebayasi 2205: IDTVEC_END(osyscall)
1.148 maxv 2206: TEXT_USER_END
2207:
2208: /*
2209: * Return to userland via 'sysret'.
2210: */
2211: TEXT_USER_BEGIN
2212: _ALIGN_TEXT
2213: LABEL(syscall_sysret)
1.190 maxv 2214: KMSAN_LEAVE
1.181 maxv 2215: MDS_LEAVE
1.148 maxv 2216: SVS_LEAVE
1.160 maxv 2217: IBRS_LEAVE
1.148 maxv 2218: INTR_RESTORE_GPRS
2219: SWAPGS
1.176 cherry 2220: #ifndef XENPV
1.148 maxv 2221: movq TF_RIP(%rsp),%rcx /* %rip for sysret */
2222: movq TF_RFLAGS(%rsp),%r11 /* %flags for sysret */
2223: movq TF_RSP(%rsp),%rsp
2224: sysretq
2225: #else
2226: addq $TF_RIP,%rsp
2227: pushq $256 /* VGCF_IN_SYSCALL */
2228: jmp HYPERVISOR_iret
2229: #endif
2230: END(syscall_sysret)
2231: TEXT_USER_END
1.26 ad 2232:
2233: /*
1.66 chs 2234: * bool sse2_idlezero_page(void *pg)
1.26 ad 2235: *
1.47 ad 2236: * Zero a page without polluting the cache. Preemption must be
2237: * disabled by the caller. Abort if a preemption is pending.
1.66 chs 2238: * Returns true if the page is zeroed, false if not.
1.26 ad 2239: */
1.47 ad 2240: ENTRY(sse2_idlezero_page)
2241: pushq %rbp
2242: movq %rsp,%rbp
2243: movl $(PAGE_SIZE/64), %ecx
1.26 ad 2244: xorq %rax, %rax
2245: .align 16
2246: 1:
1.191 ad 2247: cmpl $0, CPUVAR(RESCHED)
1.47 ad 2248: jnz 2f
1.26 ad 2249: movnti %rax, 0(%rdi)
2250: movnti %rax, 8(%rdi)
2251: movnti %rax, 16(%rdi)
2252: movnti %rax, 24(%rdi)
2253: movnti %rax, 32(%rdi)
2254: movnti %rax, 40(%rdi)
2255: movnti %rax, 48(%rdi)
2256: movnti %rax, 56(%rdi)
1.47 ad 2257: addq $64, %rdi
2258: decl %ecx
1.26 ad 2259: jnz 1b
2260: sfence
1.47 ad 2261: incl %eax
2262: popq %rbp
1.190 maxv 2263: KMSAN_INIT_RET(1)
1.26 ad 2264: ret
1.47 ad 2265: 2:
1.26 ad 2266: sfence
1.47 ad 2267: popq %rbp
1.190 maxv 2268: KMSAN_INIT_RET(1)
1.26 ad 2269: ret
1.71 uebayasi 2270: END(sse2_idlezero_page)
1.66 chs 2271:
2272: /*
2273: * void pagezero(vaddr_t va)
2274: *
1.192 ad 2275: * Zero a page.
1.66 chs 2276: */
2277: ENTRY(pagezero)
1.192 ad 2278: pushq %rbp
2279: movq %rsp,%rbp
2280: movq $(PAGE_SIZE / 8),%rcx
1.66 chs 2281: xorq %rax,%rax
1.192 ad 2282: rep
2283: stosq
2284: leave
1.66 chs 2285: ret
1.71 uebayasi 2286: END(pagezero)
1.129 maxv 2287:
1.148 maxv 2288: TEXT_USER_BEGIN
2289:
1.157 maxv 2290: /*
2291: * In intrfastexit, we advance %rsp at the beginning. We then access the
2292: * segment registers in the trapframe with TF_BACKW (backwards). See the
2293: * documentation in amd64_trap.S for an explanation.
2294: */
2295:
1.159 maxv 2296: #define TF_BACKW(val, reg) (val - (TF_REGSIZE+16))(reg)
1.157 maxv 2297:
1.148 maxv 2298: _ALIGN_TEXT
1.184 maxv 2299: .type intrfastexit,@function
1.148 maxv 2300: LABEL(intrfastexit)
1.147 maxv 2301: NOT_XEN(cli;)
1.190 maxv 2302: KMSAN_LEAVE
1.185 maxv 2303:
2304: testb $SEL_UPL,TF_CS(%rsp)
2305: jz .Lkexit
2306:
1.181 maxv 2307: MDS_LEAVE
1.147 maxv 2308: SVS_LEAVE
1.160 maxv 2309: IBRS_LEAVE
1.129 maxv 2310: INTR_RESTORE_GPRS
1.159 maxv 2311: addq $(TF_REGSIZE+16),%rsp /* iret frame */
1.185 maxv 2312: SWAPGS
1.157 maxv 2313:
2314: cmpw $LSEL(LUCODE_SEL, SEL_UPL),TF_BACKW(TF_CS, %rsp)
1.185 maxv 2315: je do_iret
1.157 maxv 2316: cmpw $GSEL(GUCODE_SEL, SEL_UPL),TF_BACKW(TF_CS, %rsp)
1.185 maxv 2317: je do_iret
1.176 cherry 2318: #ifdef XENPV
1.157 maxv 2319: cmpw $FLAT_RING3_CS64,TF_BACKW(TF_CS, %rsp)
1.185 maxv 2320: je do_iret
1.139 maxv 2321: #endif
1.129 maxv 2322:
1.138 maxv 2323: do_mov_es:
1.157 maxv 2324: movw TF_BACKW(TF_ES, %rsp),%es
1.138 maxv 2325: do_mov_ds:
1.157 maxv 2326: movw TF_BACKW(TF_DS, %rsp),%ds
1.138 maxv 2327: do_mov_fs:
1.157 maxv 2328: movw TF_BACKW(TF_FS, %rsp),%fs
1.176 cherry 2329: #ifndef XENPV
1.138 maxv 2330: do_mov_gs:
1.157 maxv 2331: movw TF_BACKW(TF_GS, %rsp),%gs
1.137 maxv 2332: #endif
1.134 maxv 2333:
1.185 maxv 2334: do_iret:
2335: iretq
1.129 maxv 2336:
2337: .Lkexit:
1.185 maxv 2338: INTR_RESTORE_GPRS
2339: addq $(TF_REGSIZE+16),%rsp /* iret frame */
1.129 maxv 2340: iretq
2341: END(intrfastexit)
1.152 maxv 2342:
2343: TEXT_USER_END
2344:
2345: #ifdef SVS
1.153 maxv 2346: .globl svs_enter, svs_enter_end
2347: .globl svs_enter_altstack, svs_enter_altstack_end
2348: .globl svs_leave, svs_leave_end
2349: .globl svs_leave_altstack, svs_leave_altstack_end
1.152 maxv 2350:
2351: LABEL(svs_enter)
1.164 joerg 2352: movabs SVS_UTLS+UTLS_KPDIRPA,%rax
1.152 maxv 2353: movq %rax,%cr3
2354: movq CPUVAR(KRSP0),%rsp
2355: LABEL(svs_enter_end)
2356:
2357: LABEL(svs_enter_altstack)
2358: testb $SEL_UPL,TF_CS(%rsp)
2359: jz 1234f
1.164 joerg 2360: movabs SVS_UTLS+UTLS_KPDIRPA,%rax
1.152 maxv 2361: movq %rax,%cr3
2362: 1234:
2363: LABEL(svs_enter_altstack_end)
2364:
1.170 maxv 2365: LABEL(svs_enter_nmi)
2366: movq %cr3,%rax
2367: movq %rax,(FRAMESIZE+1*8)(%rsp) /* nmistore->scratch */
2368: movq (FRAMESIZE+0*8)(%rsp),%rax /* nmistore->cr3 */
2369: movq %rax,%cr3
2370: LABEL(svs_enter_nmi_end)
2371:
1.152 maxv 2372: LABEL(svs_leave)
2373: movq CPUVAR(URSP0),%rsp
2374: movq CPUVAR(UPDIRPA),%rax
2375: movq %rax,%cr3
2376: LABEL(svs_leave_end)
2377:
2378: LABEL(svs_leave_altstack)
2379: testb $SEL_UPL,TF_CS(%rsp)
2380: jz 1234f
2381: movq CPUVAR(UPDIRPA),%rax
2382: movq %rax,%cr3
2383: 1234:
2384: LABEL(svs_leave_altstack_end)
1.153 maxv 2385:
1.170 maxv 2386: LABEL(svs_leave_nmi)
2387: movq (FRAMESIZE+1*8)(%rsp),%rax /* nmistore->scratch */
2388: movq %rax,%cr3
2389: LABEL(svs_leave_nmi_end)
1.152 maxv 2390: #endif
1.160 maxv 2391:
2392: .globl ibrs_enter, ibrs_enter_end
2393: .globl ibrs_leave, ibrs_leave_end
2394:
2395: /* IBRS <- 1 */
2396: LABEL(ibrs_enter)
2397: movl $MSR_IA32_SPEC_CTRL,%ecx
1.185 maxv 2398: rdmsr
2399: orl $IA32_SPEC_CTRL_IBRS,%eax
1.160 maxv 2400: wrmsr
2401: LABEL(ibrs_enter_end)
2402:
2403: /* IBRS <- 0 */
2404: LABEL(ibrs_leave)
2405: movl $MSR_IA32_SPEC_CTRL,%ecx
1.185 maxv 2406: rdmsr
2407: andl $~IA32_SPEC_CTRL_IBRS,%eax
1.160 maxv 2408: wrmsr
2409: LABEL(ibrs_leave_end)
1.162 maxv 2410:
2411: LABEL(noibrs_enter)
2412: NOIBRS_ENTER
2413: LABEL(noibrs_enter_end)
2414:
2415: LABEL(noibrs_leave)
2416: NOIBRS_LEAVE
2417: LABEL(noibrs_leave_end)
1.181 maxv 2418:
2419: .globl mds_leave, mds_leave_end
2420:
2421: LABEL(mds_leave)
2422: pushq $GSEL(GDATA_SEL, SEL_KPL)
2423: verw (%rsp)
2424: addq $8,%rsp
2425: LABEL(mds_leave_end)
2426:
2427: LABEL(nomds_leave)
2428: NOMDS_LEAVE
2429: LABEL(nomds_leave_end)
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to locore.S CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / amd64 / amd64