Annotation of src/sys/arch/amd64/amd64/locore.S, Revision 1.76.8.2
1.76.8.2! snj 1: /* $NetBSD: locore.S,v 1.76.8.1 2017/04/26 14:51:58 martin Exp $ */
1.1 fvdl 2:
3: /*
4: * Copyright-o-rama!
5: */
6:
7: /*
1.32 bouyer 8: * Copyright (c) 2007 Manuel Bouyer.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: *
19: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29: *
30: */
31:
32: /*
33: * Copyright (c) 2006 Mathieu Ropert <mro@adviseo.fr>
34: *
35: * Permission to use, copy, modify, and distribute this software for any
36: * purpose with or without fee is hereby granted, provided that the above
37: * copyright notice and this permission notice appear in all copies.
38: *
39: * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
40: * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
41: * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
42: * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
43: * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
44: * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
45: * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
46: */
47:
48: /*
1.1 fvdl 49: * Copyright (c) 2001 Wasabi Systems, Inc.
50: * All rights reserved.
51: *
52: * Written by Frank van der Linden for Wasabi Systems, Inc.
53: *
54: * Redistribution and use in source and binary forms, with or without
55: * modification, are permitted provided that the following conditions
56: * are met:
57: * 1. Redistributions of source code must retain the above copyright
58: * notice, this list of conditions and the following disclaimer.
59: * 2. Redistributions in binary form must reproduce the above copyright
60: * notice, this list of conditions and the following disclaimer in the
61: * documentation and/or other materials provided with the distribution.
62: * 3. All advertising materials mentioning features or use of this software
63: * must display the following acknowledgement:
64: * This product includes software developed for the NetBSD Project by
65: * Wasabi Systems, Inc.
66: * 4. The name of Wasabi Systems, Inc. may not be used to endorse
67: * or promote products derived from this software without specific prior
68: * written permission.
69: *
70: * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
71: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
72: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
73: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
74: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
75: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
76: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
77: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
78: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
79: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
80: * POSSIBILITY OF SUCH DAMAGE.
81: */
82:
83:
84: /*-
1.41 ad 85: * Copyright (c) 1998, 2000, 2007, 2008 The NetBSD Foundation, Inc.
1.1 fvdl 86: * All rights reserved.
87: *
88: * This code is derived from software contributed to The NetBSD Foundation
89: * by Charles M. Hannum.
90: *
91: * Redistribution and use in source and binary forms, with or without
92: * modification, are permitted provided that the following conditions
93: * are met:
94: * 1. Redistributions of source code must retain the above copyright
95: * notice, this list of conditions and the following disclaimer.
96: * 2. Redistributions in binary form must reproduce the above copyright
97: * notice, this list of conditions and the following disclaimer in the
98: * documentation and/or other materials provided with the distribution.
99: *
100: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
101: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
102: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
103: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
104: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
105: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
106: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
107: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
108: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
109: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
110: * POSSIBILITY OF SUCH DAMAGE.
111: */
112:
113: /*-
114: * Copyright (c) 1990 The Regents of the University of California.
115: * All rights reserved.
116: *
117: * This code is derived from software contributed to Berkeley by
118: * William Jolitz.
119: *
120: * Redistribution and use in source and binary forms, with or without
121: * modification, are permitted provided that the following conditions
122: * are met:
123: * 1. Redistributions of source code must retain the above copyright
124: * notice, this list of conditions and the following disclaimer.
125: * 2. Redistributions in binary form must reproduce the above copyright
126: * notice, this list of conditions and the following disclaimer in the
127: * documentation and/or other materials provided with the distribution.
1.5 agc 128: * 3. Neither the name of the University nor the names of its contributors
1.1 fvdl 129: * may be used to endorse or promote products derived from this software
130: * without specific prior written permission.
131: *
132: * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
133: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
134: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
135: * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
136: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
137: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
138: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
139: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
140: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
141: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
142: * SUCH DAMAGE.
143: *
144: * @(#)locore.s 7.3 (Berkeley) 5/13/91
145: */
146:
1.34 dsl 147: /*
148: * override user-land alignment before including asm.h
149: */
150: #define ALIGN_DATA .align 8
151: #define ALIGN_TEXT .align 16,0x90
152: #define _ALIGN_TEXT ALIGN_TEXT
153:
154: #include <machine/asm.h>
155:
1.1 fvdl 156: #include "opt_ddb.h"
157: #include "opt_ddbparam.h"
1.51 apb 158: #include "opt_modular.h"
1.1 fvdl 159: #include "opt_realmem.h"
160:
1.12 drochner 161: #include "opt_compat_netbsd.h"
162: #include "opt_compat_netbsd32.h"
163: #include "opt_compat_ibcs2.h"
1.32 bouyer 164: #include "opt_xen.h"
1.12 drochner 165:
1.1 fvdl 166: #include "assym.h"
167: #include "lapic.h"
168: #include "ioapic.h"
1.2 fvdl 169: #include "ksyms.h"
1.1 fvdl 170:
171: #include <sys/errno.h>
172: #include <sys/syscall.h>
173:
174: #include <machine/pte.h>
175: #include <machine/segments.h>
176: #include <machine/specialreg.h>
177: #include <machine/trap.h>
178: #include <machine/bootinfo.h>
179: #include <machine/frameasm.h>
1.44 ad 180: #include <machine/cputypes.h>
1.1 fvdl 181:
182: #if NLAPIC > 0
183: #include <machine/i82489reg.h>
184: #endif
185:
186: /* XXX temporary kluge; these should not be here */
187: /* Get definitions for IOM_BEGIN, IOM_END, and IOM_SIZE */
188: #include <dev/isa/isareg.h>
189:
1.71 uebayasi 190: #define _RELOC(x) ((x) - KERNBASE)
191: #define RELOC(x) _RELOC(_C_LABEL(x))
192:
1.32 bouyer 193: #ifdef XEN
194:
195: /*
1.73 uebayasi 196: * Xen guest identifier and loader selection
1.32 bouyer 197: */
198:
199: .section __xen_guest
200:
1.73 uebayasi 201: .ascii "GUEST_OS=NetBSD,GUEST_VER=4.99"
202: .ascii ",XEN_VER=xen-3.0"
203: .ascii ",LOADER=generic"
1.32 bouyer 204: .ascii ",VIRT_BASE=0xffffffff80000000"
205: .ascii ",ELF_PADDR_OFFSET=0xffffffff80000000"
206: .ascii ",VIRT_ENTRY=0xffffffff80100000"
207: .ascii ",HYPERCALL_PAGE=0x00000101" /* (???+HYPERCALL_PAGE_OFFSET)/PAGE_SIZE) */
1.49 ad 208: #if NKSYMS > 0 || defined(DDB) || defined(MODULAR)
1.33 bouyer 209: .ascii ",BSD_SYMTAB=yes"
1.32 bouyer 210: #endif
1.73 uebayasi 211: .byte 0
1.32 bouyer 212:
213: #endif /* XEN */
1.1 fvdl 214:
215: /*
216: * Initialization
217: */
218: .data
219:
1.73 uebayasi 220: #if NLAPIC > 0
221: .align PAGE_SIZE
1.71 uebayasi 222: .globl _C_LABEL(local_apic)
223: .globl _C_LABEL(lapic_id)
224: .globl _C_LABEL(lapic_tpr)
225:
226: .type _C_LABEL(local_apic), @object
227: LABEL(local_apic)
1.73 uebayasi 228: .space LAPIC_ID
1.71 uebayasi 229: END(local_apic)
230: .type _C_LABEL(lapic_id), @object
231: LABEL(lapic_id)
1.73 uebayasi 232: .long 0x00000000
233: .space LAPIC_TPRI-(LAPIC_ID+4)
1.71 uebayasi 234: END(lapic_id)
235: .type _C_LABEL(lapic_tpr), @object
236: LABEL(lapic_tpr)
1.73 uebayasi 237: .space LAPIC_PPRI-LAPIC_TPRI
1.71 uebayasi 238: END(lapic_tpr)
239: .type _C_LABEL(lapic_ppr), @object
1.1 fvdl 240: _C_LABEL(lapic_ppr):
1.73 uebayasi 241: .space LAPIC_ISR-LAPIC_PPRI
1.71 uebayasi 242: END(lapic_ppr)
243: .type _C_LABEL(lapic_isr), @object
1.1 fvdl 244: _C_LABEL(lapic_isr):
1.73 uebayasi 245: .space PAGE_SIZE-LAPIC_ISR
1.71 uebayasi 246: END(lapic_isr)
1.1 fvdl 247: #endif
248:
1.71 uebayasi 249: .globl _C_LABEL(cpu_id)
250: .globl _C_LABEL(cpu_vendorname)
251: .globl _C_LABEL(cpu_brand_id)
1.57 jym 252: .globl _C_LABEL(cpuid_level)
1.71 uebayasi 253: .globl _C_LABEL(esym)
254: .globl _C_LABEL(eblob)
255: .globl _C_LABEL(boothowto)
256: .globl _C_LABEL(bootinfo)
257: .globl _C_LABEL(atdevbase)
1.54 matt 258: .globl _C_LABEL(PDPpaddr)
1.71 uebayasi 259: .globl _C_LABEL(biosbasemem)
260: .globl _C_LABEL(biosextmem)
261: .globl _C_LABEL(gdtstore)
262: .globl _C_LABEL(cputype)
1.43 ad 263:
1.71 uebayasi 264: .type _C_LABEL(cputype), @object
1.73 uebayasi 265: LABEL(cputype) .long 0 # are we 80486, Pentium, or..
1.71 uebayasi 266: END(cputype)
267: .type _C_LABEL(cpu_id), @object
268: LABEL(cpu_id) .long 0 # saved from `cpuid' instruction
269: END(cpu_id)
270: .type _C_LABEL(cpuid_level), @object
271: LABEL(cpuid_level) .long -1 # max. level accepted by 'cpuid'
1.1 fvdl 272: # instruction
1.71 uebayasi 273: END(cpuid_level)
274: .type _C_LABEL(cpu_vendorname), @object
275: LABEL(cpu_vendorname) .space 16 # vendor string returned by `cpuid'
1.1 fvdl 276: # instruction
1.71 uebayasi 277: END(cpu_vendorname)
278: .type _C_LABEL(cpu_brand_id), @object
279: LABEL(cpu_brand_id) .long 0 # brand ID from 'cpuid' instruction
280: END(cpu_brand_id)
281: .type _C_LABEL(esym), @object
282: LABEL(esym) .quad 0 # ptr to end of syms
283: END(esym)
284: .type _C_LABEL(eblob), @object
285: LABEL(eblob) .quad 0 # ptr to end of modules
286: END(eblob)
287: .type _C_LABEL(atdevbase), @object
288: LABEL(atdevbase) .quad 0 # location of start of iomem in virtual
289: END(atdevbase)
290: .type _C_LABEL(PDPpaddr), @object
291: LABEL(PDPpaddr) .quad 0 # paddr of PTD, for libkvm
292: END(PDPpaddr)
293: .type _C_LABEL(biosbasemem), @object
1.1 fvdl 294: #ifndef REALBASEMEM
1.71 uebayasi 295: LABEL(biosbasemem) .long 0 # base memory reported by BIOS
1.1 fvdl 296: #else
1.71 uebayasi 297: LABEL(biosbasemem) .long REALBASEMEM
1.1 fvdl 298: #endif
1.71 uebayasi 299: END(biosbasemem)
300: .type _C_LABEL(biosextmem), @object
1.1 fvdl 301: #ifndef REALEXTMEM
1.71 uebayasi 302: LABEL(biosextmem) .long 0 # extended memory reported by BIOS
1.1 fvdl 303: #else
1.71 uebayasi 304: LABEL(biosextmem) .long REALEXTMEM
1.1 fvdl 305: #endif
1.71 uebayasi 306: END(biosextmem)
1.1 fvdl 307:
1.32 bouyer 308: #ifndef XEN
1.56 jym 309: .globl gdt64_lo
310: .globl gdt64_hi
1.1 fvdl 311:
1.56 jym 312: #define GDT64_LIMIT gdt64_end-gdt64_start-1
313:
314: /* Temporary gdt64, with base address in low memory */
1.71 uebayasi 315: .type _C_LABEL(gdt64_lo), @object
316: LABEL(gdt64_lo)
1.56 jym 317: .word GDT64_LIMIT
1.1 fvdl 318: .quad _RELOC(gdt64_start)
1.71 uebayasi 319: END(gdt64_lo)
1.1 fvdl 320: .align 64
321:
1.56 jym 322: /* Temporary gdt64, with base address in high memory */
1.71 uebayasi 323: .type _C_LABEL(gdt64_hi), @object
324: LABEL(gdt64_hi)
1.56 jym 325: .word GDT64_LIMIT
326: .quad gdt64_start
1.71 uebayasi 327: END(gdt64_hi)
1.56 jym 328: .align 64
329:
330: #undef GDT64_LIMIT
331:
1.71 uebayasi 332: .type _C_LABEL(gdt64_start), @object
333: _C_LABEL(gdt64_start):
1.1 fvdl 334: .quad 0x0000000000000000 /* always empty */
335: .quad 0x00af9a000000ffff /* kernel CS */
336: .quad 0x00cf92000000ffff /* kernel DS */
1.71 uebayasi 337: END(gdt64_start)
1.1 fvdl 338: gdt64_end:
339:
1.71 uebayasi 340: .type _C_LABEL(farjmp64), @object
341: _C_LABEL(farjmp64):
1.57 jym 342: .long _RELOC(longmode)
1.1 fvdl 343: .word GSEL(GCODE_SEL, SEL_KPL)
1.71 uebayasi 344: END(farjmp64)
1.1 fvdl 345:
1.32 bouyer 346: #endif /* !XEN */
1.71 uebayasi 347:
348: .size tmpstk, tmpstk - .
1.1 fvdl 349: .space 512
350: tmpstk:
351:
352: .globl _C_LABEL(cpu_private)
1.73 uebayasi 353: .comm _C_LABEL(cpu_private),PAGE_SIZE,PAGE_SIZE
1.1 fvdl 354:
355: /*
356: * Some hackage to deal with 64bit symbols in 32 bit mode.
357: * This may not be needed it things are cleaned up a little.
358: */
359:
360:
361: .text
362: .globl _C_LABEL(kernel_text)
363: .set _C_LABEL(kernel_text),KERNTEXTOFF
364:
1.71 uebayasi 365: ENTRY(start)
1.32 bouyer 366: #ifndef XEN
1.71 uebayasi 367: .code32
368: movw $0x1234,0x472 # warm boot
1.1 fvdl 369: /*
370: * Load parameters from stack
371: * (howto, [bootdev], bootinfo, esym, basemem, extmem).
372: */
373: movl 4(%esp),%eax
374: movl %eax,RELOC(boothowto)
375: movl 12(%esp),%eax
376: testl %eax, %eax
377: jz 1f
378: movl (%eax), %ebx /* number of entries */
379: movl $RELOC(bootinfo),%ebp
380: movl %ebp, %edx
381: addl $BOOTINFO_MAXSIZE,%ebp
382: movl %ebx, (%edx)
383: addl $4, %edx
384: 2:
385: testl %ebx, %ebx
386: jz 1f
387: addl $4, %eax
388: movl (%eax), %ecx /* address of entry */
389: pushl %edi
390: pushl %esi
391: pushl %eax
392:
393: movl (%ecx),%eax /* len */
394: movl %edx,%edi
395: addl (%ecx), %edx /* update dest pointer */
396: cmpl %ebp, %edx
397: jg 2f
398: movl %ecx,%esi
399: movl %eax,%ecx
1.43 ad 400: /*
401: * If any modules were loaded, record where they
402: * end. We'll need to skip over them.
403: */
404: cmpl $BTINFO_MODULELIST, 4(%esi)
405: jne 0f
406: pushl 12(%esi) /* endpa */
407: popl RELOC(eblob)
408: addl $KERNBASE_LO, RELOC(eblob)
409: adcl $KERNBASE_HI, RELOC(eblob)+4
410: 0:
1.1 fvdl 411: rep
412: movsb
413: popl %eax
414: popl %esi
415: popl %edi
416: subl $1, %ebx
417: jmp 2b
418: 2: /* cleanup for overflow case */
419: popl %eax
420: popl %esi
421: popl %edi
422: movl $RELOC(bootinfo),%ebp
423: movl %ebp, %edx
424: subl %ebx, (%edx) /* correct number of entries */
425: 1:
426:
1.72 uebayasi 427: movl 16(%esp),%eax
1.1 fvdl 428: testl %eax,%eax
429: jz 1f
430: addl $KERNBASE_LO,%eax
431: 1: movl $RELOC(esym),%ebp
432: movl %eax,(%ebp)
433: movl $KERNBASE_HI,4(%ebp)
434:
435: movl $RELOC(biosextmem),%ebp
436: movl (%ebp),%eax
437: testl %eax,%eax
438: jnz 1f
439: movl 20(%esp),%eax
440: movl %eax,(%ebp)
441: 1:
442: movl $RELOC(biosbasemem),%ebp
443: movl (%ebp),%eax
444: testl %eax,%eax
445: jnz 1f
446: movl 24(%esp),%eax
447: movl %eax,(%ebp)
448: 1:
449:
450: /* First, reset the PSL. */
451: pushl $PSL_MBO
452: popfl
453:
454: xorl %eax,%eax
455: cpuid
456: movl %eax,RELOC(cpuid_level)
1.30 ad 457: movl $RELOC(cpu_vendorname),%ebp
1.1 fvdl 458: movl %ebx,(%ebp)
459: movl %edx,4(%ebp)
460: movl %ecx,8(%ebp)
1.73 uebayasi 461: movl $0, 12(%ebp)
1.1 fvdl 462:
463: movl $1,%eax
464: cpuid
465: movl %eax,RELOC(cpu_id)
466:
467: /* Brand ID is bits 0-7 of %ebx */
468: andl $255,%ebx
469: movl %ebx,RELOC(cpu_brand_id)
470:
471: /*
472: * Finished with old stack; load new %esp now instead of later so we
473: * can trace this code without having to worry about the trace trap
474: * clobbering the memory test or the zeroing of the bss+bootstrap page
475: * tables.
476: *
477: * The boot program should check:
478: * text+data <= &stack_variable - more_space_for_stack
479: * text+data+bss+pad+space_for_page_tables <= end_of_memory
480: * Oops, the gdt is in the carcass of the boot program so clearing
481: * the rest of memory is still not possible.
482: */
483: movl $RELOC(tmpstk),%esp
484:
485: /*
486: * Virtual address space of kernel:
487: *
1.73 uebayasi 488: * text | data | bss | [syms] | page dir | proc0 kstack | L1 ptp | L2 ptp | L3
1.1 fvdl 489: * 0 1 2 3
490: */
491:
492: #if L2_SLOT_KERNBASE > 0
493: #define TABLE_L2_ENTRIES (2 * (NKL2_KIMG_ENTRIES + 1))
494: #else
495: #define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1)
496: #endif
497:
498: #if L3_SLOT_KERNBASE > 0
499: #define TABLE_L3_ENTRIES (2 * NKL3_KIMG_ENTRIES)
500: #else
501: #define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES
502: #endif
503:
504:
505: #define PROC0_PML4_OFF 0
1.73 uebayasi 506: #define PROC0_STK_OFF (PROC0_PML4_OFF + PAGE_SIZE)
507: #define PROC0_PTP3_OFF (PROC0_STK_OFF + UPAGES * PAGE_SIZE)
508: #define PROC0_PTP2_OFF (PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * PAGE_SIZE)
509: #define PROC0_PTP1_OFF (PROC0_PTP2_OFF + TABLE_L3_ENTRIES * PAGE_SIZE)
1.1 fvdl 510: #define TABLESIZE \
511: ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \
1.73 uebayasi 512: * PAGE_SIZE)
1.1 fvdl 513:
514: #define fillkpt \
1.73 uebayasi 515: 1: movl %eax,(%ebx) ; /* store phys addr */ \
516: movl $0,4(%ebx) ; /* upper 32 bits 0 */ \
517: addl $8,%ebx ; /* next pte/pde */ \
518: addl $PAGE_SIZE,%eax ; /* next phys page */ \
519: loop 1b ;
1.1 fvdl 520:
521: /* Find end of kernel image. */
522: movl $RELOC(end),%edi
1.49 ad 523: #if (NKSYMS || defined(DDB) || defined(MODULAR)) && !defined(SYMTAB_SPACE)
1.1 fvdl 524: /* Save the symbols (if loaded). */
525: movl RELOC(esym),%eax
526: testl %eax,%eax
527: jz 1f
528: subl $KERNBASE_LO,%eax /* XXX */
529: movl %eax,%edi
530: 1:
531: #endif
1.43 ad 532: /* Skip over any modules/blobs. */
533: movl RELOC(eblob),%eax
534: testl %eax,%eax
535: jz 1f
536: subl $KERNBASE_LO,%eax /* XXX */
537: movl %eax,%edi
538: 1:
1.73 uebayasi 539: /* Compute sizes */
1.1 fvdl 540: movl %edi,%esi
1.73 uebayasi 541: addl $PGOFSET,%esi # page align up
1.1 fvdl 542: andl $~PGOFSET,%esi
543:
544: movl %esi,%edi
545: xorl %eax,%eax
546: cld
547: movl $TABLESIZE,%ecx
548: shrl $2,%ecx
549: rep
550: stosl
551:
552: leal (PROC0_PTP1_OFF)(%esi), %ebx
553:
1.73 uebayasi 554: /*
555: * Build initial page tables.
556: */
1.1 fvdl 557: /*
1.72 uebayasi 558: * Compute &__data_start - KERNBASE. This can't be > 4G,
1.29 yamt 559: * or we can't deal with it anyway, since we can't load it in
560: * 32 bit mode. So use the bottom 32 bits.
1.1 fvdl 561: */
1.29 yamt 562: movl $RELOC(__data_start),%edx
1.1 fvdl 563: andl $~PGOFSET,%edx
564:
565: /*
566: * Skip the first MB.
567: */
568: movl $(KERNTEXTOFF_LO - KERNBASE_LO),%eax
569: movl %eax,%ecx
570: shrl $(PGSHIFT-3),%ecx /* ((n >> PGSHIFT) << 3) for # pdes */
571: addl %ecx,%ebx
572:
1.73 uebayasi 573: /* Map the kernel text read-only. */
1.1 fvdl 574: movl %edx,%ecx
575: subl %eax,%ecx
576: shrl $PGSHIFT,%ecx
1.73 uebayasi 577: orl $(PG_V|PG_KR),%eax
1.1 fvdl 578: fillkpt
579:
580: /* Map the data, BSS, and bootstrap tables read-write. */
581: leal (PG_V|PG_KW)(%edx),%eax
582: movl $TABLESIZE,%ecx
1.73 uebayasi 583: addl %esi,%ecx # end of tables
584: subl %edx,%ecx # subtract end of text
1.1 fvdl 585: shrl $PGSHIFT,%ecx
586: fillkpt
587:
588: /* Map ISA I/O mem (later atdevbase) */
1.73 uebayasi 589: movl $(IOM_BEGIN|PG_V|PG_KW/*|PG_N*/),%eax # having these bits set
590: movl $(IOM_SIZE>>PGSHIFT),%ecx # for this many pte s,
1.1 fvdl 591: fillkpt
592:
1.73 uebayasi 593: /*
594: * Construct a page table directory.
595: */
1.1 fvdl 596: /* Set up level 2 pages */
1.73 uebayasi 597: leal (PROC0_PTP2_OFF)(%esi),%ebx
1.1 fvdl 598: leal (PROC0_PTP1_OFF)(%esi),%eax
599: orl $(PG_V|PG_KW), %eax
600: movl $(NKL2_KIMG_ENTRIES+1),%ecx
601: fillkpt
602:
603: #if L2_SLOT_KERNBASE > 0
604: /* If needed, set up level 2 entries for actual kernel mapping */
605: leal (PROC0_PTP2_OFF+ L2_SLOT_KERNBASE*8)(%esi),%ebx
1.73 uebayasi 606: leal (PROC0_PTP1_OFF)(%esi),%eax
607: orl $(PG_V|PG_KW), %eax
608: movl $(NKL2_KIMG_ENTRIES+1),%ecx
1.1 fvdl 609: fillkpt
610: #endif
611:
612: /* Set up level 3 pages */
1.73 uebayasi 613: leal (PROC0_PTP3_OFF)(%esi),%ebx
1.1 fvdl 614: leal (PROC0_PTP2_OFF)(%esi),%eax
615: orl $(PG_V|PG_KW), %eax
616: movl $NKL3_KIMG_ENTRIES,%ecx
617: fillkpt
618:
619: #if L3_SLOT_KERNBASE > 0
620: /* If needed, set up level 3 entries for actual kernel mapping */
621: leal (PROC0_PTP3_OFF+ L3_SLOT_KERNBASE*8)(%esi),%ebx
1.73 uebayasi 622: leal (PROC0_PTP2_OFF)(%esi),%eax
623: orl $(PG_V|PG_KW), %eax
624: movl $NKL3_KIMG_ENTRIES,%ecx
1.1 fvdl 625: fillkpt
626: #endif
627:
628: /* Set up top level entries for identity mapping */
1.73 uebayasi 629: leal (PROC0_PML4_OFF)(%esi),%ebx
1.1 fvdl 630: leal (PROC0_PTP3_OFF)(%esi),%eax
631: orl $(PG_V|PG_KW), %eax
632: movl $NKL4_KIMG_ENTRIES,%ecx
633: fillkpt
634:
635: /* Set up top level entries for actual kernel mapping */
1.73 uebayasi 636: leal (PROC0_PML4_OFF + L4_SLOT_KERNBASE*8)(%esi),%ebx
1.1 fvdl 637: leal (PROC0_PTP3_OFF)(%esi),%eax
638: orl $(PG_V|PG_KW), %eax
639: movl $NKL4_KIMG_ENTRIES,%ecx
640: fillkpt
641:
642: /* Install recursive top level PDE */
1.73 uebayasi 643: leal (PROC0_PML4_OFF + PDIR_SLOT_PTE*8)(%esi),%ebx
644: leal (PROC0_PML4_OFF)(%esi),%eax
1.1 fvdl 645: orl $(PG_V|PG_KW),%eax
646: movl %eax,(%ebx)
647: movl $0, 4(%ebx)
648:
649:
650: /* Save phys. addr of PTD, for libkvm. */
1.29 yamt 651: movl $RELOC(PDPpaddr),%ebp
1.1 fvdl 652: movl %esi,(%ebp)
653: movl $0,4(%ebp)
654:
655: /*
656: * Startup checklist:
657: * 1. Enable PAE (and SSE while here).
658: */
659: movl %cr4,%eax
660: orl $(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax
661: movl %eax,%cr4
662:
663: /*
664: * 2. Set Long Mode Enable in EFER. Also enable the
665: * syscall extensions.
666: */
1.73 uebayasi 667: movl $MSR_EFER,%ecx
1.1 fvdl 668: rdmsr
669: xorl %eax,%eax /* XXX */
670: orl $(EFER_LME|EFER_SCE),%eax
671: wrmsr
672:
673: /*
674: * 3. Load %cr3 with pointer to PML4.
675: */
676: movl %esi,%eax
677: movl %eax,%cr3
678:
679: /*
680: * 4. Enable paging and the rest of it.
681: */
682: movl %cr0,%eax
1.68 jym 683: orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax
1.1 fvdl 684: movl %eax,%cr0
685: jmp compat
686: compat:
687:
688: /*
689: * 5.
690: * Not quite done yet, we're now in a compatibility segment,
691: * in legacy mode. We must jump to a long mode segment.
692: * Need to set up a temporary GDT with a long mode segment
693: * in it to do that.
694: */
695:
1.56 jym 696: movl $RELOC(gdt64_lo),%eax
1.1 fvdl 697: lgdt (%eax)
698: movl $RELOC(farjmp64),%eax
699: ljmp *(%eax)
700:
701: .code64
702: longmode:
703: /*
704: * 6.
705: * Finally, we're in long mode. However, we're still
706: * in the identity mapped area (could not jump out
707: * of that earlier because it would have been a > 32bit
708: * jump). We can do that now, so here we go.
709: */
710: movabsq $longmode_hi,%rax
711: jmp *%rax
1.56 jym 712:
1.1 fvdl 713: longmode_hi:
1.56 jym 714:
715: /*
716: * We left the identity mapped area. Base address of
717: * the temporary gdt64 should now be in high memory.
718: */
719: movq $RELOC(gdt64_hi),%rax
720: lgdt (%rax)
721:
1.1 fvdl 722: /*
1.73 uebayasi 723: * We have arrived.
1.1 fvdl 724: * There's no need anymore for the identity mapping in low
725: * memory, remove it.
726: */
727: movq $KERNBASE,%r8
728:
729: #if L2_SLOT_KERNBASE > 0
730: movq $(NKL2_KIMG_ENTRIES+1),%rcx
731: leaq (PROC0_PTP2_OFF)(%rsi),%rbx
732: addq %r8, %rbx
733: 1: movq $0,(%rbx)
734: addq $8,%rbx
735: loop 1b
736: #endif
737:
738: #if L3_SLOT_KERNBASE > 0
739: movq $NKL3_KIMG_ENTRIES,%rcx
740: leaq (PROC0_PTP3_OFF)(%rsi),%rbx
741: addq %r8, %rbx
742: 1: movq $0,(%rbx)
743: addq $8,%rbx
744: loop 1b
745: #endif
746:
747: movq $NKL4_KIMG_ENTRIES,%rcx
748: leaq (PROC0_PML4_OFF)(%rsi),%rbx # old, phys address of PML4
1.8 wiz 749: addq %r8, %rbx # new, virtual address of PML4
1.1 fvdl 750: 1: movq $0,(%rbx)
751: addq $8,%rbx
752: loop 1b
753:
754: /* Relocate atdevbase. */
755: movq $(TABLESIZE+KERNBASE),%rdx
756: addq %rsi,%rdx
757: movq %rdx,_C_LABEL(atdevbase)(%rip)
758:
759: /* Set up bootstrap stack. */
760: leaq (PROC0_STK_OFF)(%rsi),%rax
761: addq %r8,%rax
1.55 rmind 762: movq %rax,(_C_LABEL(lwp0)+L_PCB)(%rip) /* XXX L_PCB != uarea */
1.1 fvdl 763: leaq (USPACE-FRAMESIZE)(%rax),%rsp
764: movq %rsi,PCB_CR3(%rax) # pcb->pcb_cr3
1.73 uebayasi 765: xorq %rbp,%rbp # mark end of frames
1.1 fvdl 766:
767: xorw %ax,%ax
768: movw %ax,%gs
769: movw %ax,%fs
770:
771: /* XXX merge these */
1.46 ad 772: leaq (TABLESIZE+IOM_SIZE)(%rsi),%rdi
1.32 bouyer 773:
774: #else /* XEN */
775: /* First, reset the PSL. */
776: pushq $2
777: popfq
778:
779: cld
780:
781: /*
782: * Xen info:
783: * - %rsi -> start_info struct
784: * - %rsp -> stack, *theorically* the last used page
785: * by Xen bootstrap
786: */
787: movq %rsi, %rbx
788:
1.73 uebayasi 789: /* Clear BSS. */
1.32 bouyer 790: xorq %rax,%rax
791: movq $_C_LABEL(__bss_start),%rdi
792: movq $_C_LABEL(_end),%rcx
793: subq %rdi,%rcx
794: rep
795: stosb
796:
797: /* Copy start_info to a safe place */
798: movq %rbx,%rsi
799: movq $_C_LABEL(start_info_union),%rdi
800: movq $64,%rcx
801: rep
802: movsq
803:
804: /*
805: * Memory layout at start of the day:
806: * - Kernel image
807: * - Page frames list
808: * - start_info struct. we copied it, so it can be recycled.
809: * - xenstore
810: * - console
811: * - Xen bootstrap page tables
812: * - kernel stack. provided by Xen
813: * - guaranted 512kB padding
814: *
815: * As we want to rebuild our page tables and place our stack
816: * in proc0 struct, all data starting from after console can be
817: * discarded after we've done a little setup.
818: */
819:
820: /*
821: * We want our own page tables, let's rebuild them
822: * We will reclaim xen space afterward INCLUDING stack
823: * so let's change it to a temporary one
824: */
825:
826: movq $tmpstk, %rax
827: subq $8, %rax
828: movq %rax, %rsp
829:
1.45 bouyer 830: xorl %eax,%eax
831: cpuid
832: movl %eax,_C_LABEL(cpuid_level)
833:
1.67 cherry 834: movq $cpu_info_primary, %rdi
835: movq %rdi, CPU_INFO_SELF(%rdi) /* ci->ci_self = ci */
836: movq $1, %rsi
837: call cpu_init_msrs /* cpu_init_msrs(ci, true); */
838:
1.32 bouyer 839: call xen_pmap_bootstrap
840:
841: /*
842: * First avail returned by xen_pmap_bootstrap in %rax
843: */
844: movq %rax, %rsi
1.55 rmind 845: movq %rsi,(_C_LABEL(lwp0)+L_PCB) /* XXX L_PCB != uarea */
1.32 bouyer 846:
847: /*
848: * Set new stack and clear segments
849: */
850:
851: leaq (USPACE-FRAMESIZE)(%rsi),%rsp
852: xorq %rbp,%rbp
853:
1.63 jym 854: xorw %ax,%ax
1.32 bouyer 855: movw %ax,%gs
856: movw %ax,%fs
857:
858: /*
859: * Set first_avail after proc0
860: */
861:
862: movq %rsi,%rdi
863: addq $USPACE,%rdi
864: subq $KERNBASE,%rdi # init_x86_64 want a physical address
865:
1.73 uebayasi 866: #endif /* XEN */
1.1 fvdl 867: call _C_LABEL(init_x86_64)
868:
869: call _C_LABEL(main)
1.71 uebayasi 870: END(start)
1.1 fvdl 871:
1.73 uebayasi 872: #if defined(XEN)
1.32 bouyer 873: /* space for the hypercall call page */
874: #define HYPERCALL_PAGE_OFFSET 0x1000
875: .org HYPERCALL_PAGE_OFFSET
876: ENTRY(hypercall_page)
877: .skip 0x1000
1.71 uebayasi 878: END(hypercall_page)
1.32 bouyer 879: #endif /* XEN */
1.1 fvdl 880:
881: /*
1.22 yamt 882: * int setjmp(label_t *)
883: *
884: * Used primarily by DDB.
885: */
1.1 fvdl 886: ENTRY(setjmp)
887: /*
888: * Only save registers that must be preserved across function
889: * calls according to the ABI (%rbx, %rsp, %rbp, %r12-%r15)
890: * and %rip.
891: */
892: movq %rdi,%rax
893: movq %rbx,(%rax)
894: movq %rsp,8(%rax)
895: movq %rbp,16(%rax)
896: movq %r12,24(%rax)
897: movq %r13,32(%rax)
898: movq %r14,40(%rax)
899: movq %r15,48(%rax)
900: movq (%rsp),%rdx
901: movq %rdx,56(%rax)
902: xorl %eax,%eax
903: ret
1.71 uebayasi 904: END(setjmp)
1.1 fvdl 905:
1.22 yamt 906: /*
907: * int longjmp(label_t *)
908: *
909: * Used primarily by DDB.
910: */
1.1 fvdl 911: ENTRY(longjmp)
912: movq %rdi,%rax
913: movq (%rax),%rbx
914: movq 8(%rax),%rsp
915: movq 16(%rax),%rbp
916: movq 24(%rax),%r12
917: movq 32(%rax),%r13
918: movq 40(%rax),%r14
919: movq 48(%rax),%r15
920: movq 56(%rax),%rdx
921: movq %rdx,(%rsp)
1.22 yamt 922: movl $1,%eax
1.1 fvdl 923: ret
1.71 uebayasi 924: END(longjmp)
1.1 fvdl 925:
1.73 uebayasi 926: /*
927: * void dumpsys(void)
928: *
929: * Mimic cpu_switchto() for postmortem debugging.
930: */
1.25 yamt 931: ENTRY(dumpsys)
932: # build a fake switch frame.
933: pushq %rbx
934: pushq %r12
935: pushq %r13
936: pushq %r14
937: pushq %r15
938:
939: # save a context.
940: movq $dumppcb, %rax
941: movq %rsp, PCB_RSP(%rax)
942: movq %rbp, PCB_RBP(%rax)
943:
944: call _C_LABEL(dodumpsys)
945:
946: addq $(5*8), %rsp # sizeof(switchframe) - sizeof(%rip)
947: ret
1.71 uebayasi 948: END(dumpsys)
1.25 yamt 949:
1.1 fvdl 950: /*
1.58 chs 951: * struct lwp *cpu_switchto(struct lwp *oldlwp, struct lwp *newlwp,
1.35 ad 952: * bool returning)
1.22 yamt 953: *
954: * 1. if (oldlwp != NULL), save its context.
955: * 2. then, restore context of newlwp.
956: *
957: * Note that the stack frame layout is known to "struct switchframe" in
958: * <machine/frame.h> and to the code in cpu_lwp_fork() which initializes
959: * it for a new lwp.
1.1 fvdl 960: */
1.22 yamt 961: ENTRY(cpu_switchto)
1.1 fvdl 962: pushq %rbx
963: pushq %r12
964: pushq %r13
965: pushq %r14
966: pushq %r15
967:
1.22 yamt 968: movq %rdi,%r13 # oldlwp
969: movq %rsi,%r12 # newlwp
1.1 fvdl 970:
1.22 yamt 971: testq %r13,%r13
972: jz 1f
1.1 fvdl 973:
1.22 yamt 974: /* Save old context. */
1.55 rmind 975: movq L_PCB(%r13),%rax
1.22 yamt 976: movq %rsp,PCB_RSP(%rax)
977: movq %rbp,PCB_RBP(%rax)
978:
979: /* Switch to newlwp's stack. */
1.55 rmind 980: 1: movq L_PCB(%r12),%r14
1.32 bouyer 981: #ifdef XEN /* XXX debug code */
982: cmpq $0, PCB_RSP(%r14)
983: jne 999f
984: callq _C_LABEL(cpu_Debugger);
985: 999:
986: #endif
1.22 yamt 987: movq PCB_RSP(%r14),%rsp
988: movq PCB_RBP(%r14),%rbp
1.1 fvdl 989:
1.39 ad 990: /*
991: * Set curlwp. This must be globally visible in order to permit
992: * non-interlocked mutex release.
993: */
994: movq %r12,%rcx
995: xchgq %rcx,CPUVAR(CURLWP)
1.35 ad 996:
997: /* Skip the rest if returning to a pinned LWP. */
998: testb %dl,%dl
999: jnz 4f
1000:
1.38 yamt 1001: /* Switch ring0 stack */
1002: #ifndef XEN
1003: movq PCB_RSP0(%r14),%rax
1004: movq %rax,CPUVAR(RSP0)
1005: #else
1.32 bouyer 1006: movq %r14, %rdi
1007: callq _C_LABEL(x86_64_switch_context);
1008: #endif
1.1 fvdl 1009:
1.22 yamt 1010: /* Don't bother with the rest if switching to a system process. */
1011: testl $LW_SYSTEM,L_FLAG(%r12)
1012: jnz 4f
1.1 fvdl 1013:
1.22 yamt 1014: /* Is this process using RAS (restartable atomic sequences)? */
1015: movq L_PROC(%r12),%rdi
1016: cmpq $0,P_RASLIST(%rdi)
1017: jne 5f
1.1 fvdl 1018:
1019: /*
1.76 rmind 1020: * Restore cr0 including FPU state (may have CR0_TS set). Note that
1021: * IPL_SCHED prevents from FPU interrupt altering the LWP's saved cr0.
1.1 fvdl 1022: */
1.32 bouyer 1023: 2:
1024: #ifndef XEN
1.75 christos 1025: movl $IPL_HIGH,CPUVAR(ILEVEL)
1026: movl PCB_CR0(%r14),%ecx /* has CR0_TS clear */
1.22 yamt 1027: movq %cr0,%rdx
1.1 fvdl 1028:
1.75 christos 1029: /*
1030: * If our floating point registers are on a different CPU,
1031: * set CR0_TS so we'll trap rather than reuse bogus state.
1032: */
1033: cmpq CPUVAR(FPCURLWP),%r12
1034: je 3f
1035: orq $CR0_TS,%rcx
1036:
1.22 yamt 1037: /* Reloading CR0 is very expensive - avoid if possible. */
1.75 christos 1038: 3: cmpq %rdx,%rcx
1.58 chs 1039: je 6f
1.1 fvdl 1040: movq %rcx,%cr0
1.58 chs 1041:
1.73 uebayasi 1042: 6: testl $PCB_COMPAT32, PCB_FLAGS(%r14)
1.58 chs 1043: jne 32f
1044:
1045: /* Zero out %fs/%gs registers and GDT descriptors. */
1046: xorq %rax, %rax
1047: movw %ax, %fs
1048: CLI(cx)
1.70 dsl 1049: SWAPGS
1.58 chs 1050: movw %ax, %gs
1.70 dsl 1051: SWAPGS
1.58 chs 1052: STI(cx)
1053:
1054: movq CPUVAR(GDT),%rcx
1055: movq %rax, (GUFS_SEL*8)(%rcx)
1056: movq %rax, (GUGS_SEL*8)(%rcx)
1057:
1058: /* Reload 64-bit %fs/%gs MSRs. */
1059: movl $MSR_FSBASE, %ecx
1060: movl PCB_FS(%r14), %eax
1061: movl 4+PCB_FS(%r14), %edx
1062: wrmsr
1063: movl $MSR_KERNELGSBASE, %ecx
1064: movl PCB_GS(%r14), %eax
1065: movl 4+PCB_GS(%r14), %edx
1066: wrmsr
1067: jmp 4f
1068:
1069: 32:
1070: /* Reload %fs/%gs GDT descriptors. */
1071: movq CPUVAR(GDT),%rcx
1072: movq PCB_FS(%r14), %rax
1073: movq %rax, (GUFS_SEL*8)(%rcx)
1074: movq PCB_GS(%r14), %rax
1075: movq %rax, (GUGS_SEL*8)(%rcx)
1076:
1077: /* Reload %fs and %gs */
1078: movq L_MD_REGS(%r12), %rbx
1079: movw TF_FS(%rbx), %fs
1080: CLI(ax)
1.70 dsl 1081: SWAPGS
1.58 chs 1082: movw TF_GS(%rbx), %gs
1.70 dsl 1083: SWAPGS
1.58 chs 1084: STI(ax)
1085:
1086: #else
1087: movq %r12,%rdi
1088: callq _C_LABEL(x86_64_tls_switch)
1.32 bouyer 1089: #endif
1.58 chs 1090:
1.22 yamt 1091: /* Return to the new LWP, returning 'oldlwp' in %rax. */
1092: 4: movq %r13,%rax
1.1 fvdl 1093: popq %r15
1094: popq %r14
1095: popq %r13
1096: popq %r12
1097: popq %rbx
1098: ret
1099:
1.22 yamt 1100: /* Check for restartable atomic sequences (RAS). */
1101: 5: movq L_MD_REGS(%r12),%rbx
1102: movq TF_RIP(%rbx),%rsi
1103: call _C_LABEL(ras_lookup)
1104: cmpq $-1,%rax
1105: je 2b
1106: movq %rax,TF_RIP(%rbx)
1107: jmp 2b
1.71 uebayasi 1108: END(cpu_switchto)
1.1 fvdl 1109:
1110: /*
1.22 yamt 1111: * void savectx(struct pcb *pcb);
1112: *
1.1 fvdl 1113: * Update pcb, saving current processor state.
1114: */
1115: ENTRY(savectx)
1116: /* Save stack pointers. */
1117: movq %rsp,PCB_RSP(%rdi)
1118: movq %rbp,PCB_RBP(%rdi)
1119: ret
1.71 uebayasi 1120: END(savectx)
1.1 fvdl 1121:
1122: IDTVEC(syscall32)
1123: sysret /* go away please */
1.71 uebayasi 1124: IDTVEC_END(syscall32)
1.1 fvdl 1125:
1126: /*
1.22 yamt 1127: * syscall()
1128: *
1.70 dsl 1129: * syscall insn entry.
1130: * This currently isn't much faster, but it can be made faster in the future.
1131: * (Actually we've already saved a few 100 clocks by not loading the trap gate)
1.1 fvdl 1132: */
1133: IDTVEC(syscall)
1.32 bouyer 1134: #ifndef XEN
1.70 dsl 1135: /*
1136: * The user %rip is in %rcx and the user %flags in %r11.
1137: * The kernel %cs and %ss are loaded, but nothing else is.
1138: * The 'swapgs' gives us access to cpu-specific memory where
1139: * we can save a user register and then read the lwps
1140: * kernel stack pointer,
1141: * This code doesn't seem to set %ds, this may not matter since it
1142: * is ignored in 64bit mode, OTOH the syscall instruction sets %ss
1143: * and that is ignored as well.
1144: */
1.1 fvdl 1145: swapgs
1146: movq %r15,CPUVAR(SCRATCH)
1.22 yamt 1147: movq CPUVAR(CURLWP),%r15
1.55 rmind 1148: movq L_PCB(%r15),%r15
1.70 dsl 1149: movq PCB_RSP0(%r15),%r15 /* LWP's kernel stack pointer */
1150:
1151: /* Make stack look like an 'int nn' frame */
1152: #define SP(x) (x)-(TF_SS+8)(%r15)
1153: movq $(LSEL(LUDATA_SEL, SEL_UPL)), SP(TF_SS) /* user %ss */
1154: movq %rsp, SP(TF_RSP) /* User space rsp */
1155:
1156: movq %r11, SP(TF_RFLAGS) /* old rflags from syscall insn */
1157: movq $(LSEL(LUCODE_SEL, SEL_UPL)), SP(TF_CS)
1158: movq %rcx, SP(TF_RIP) /* syscall saves rip in rcx */
1.1 fvdl 1159:
1.70 dsl 1160: leaq SP(0),%rsp /* %rsp now valid after frame */
1.1 fvdl 1161: movq CPUVAR(SCRATCH),%r15
1.70 dsl 1162: #undef SP
1163:
1164: movq $2,TF_ERR(%rsp) /* syscall instruction size */
1165: movq $T_ASTFLT, TF_TRAPNO(%rsp)
1166:
1.59 yamt 1167: movw %es,TF_ES(%rsp)
1.41 ad 1168: sti
1.1 fvdl 1169: INTR_SAVE_GPRS
1.24 fvdl 1170: movw %fs,TF_FS(%rsp)
1171: movw %gs,TF_GS(%rsp)
1.1 fvdl 1172: movw $(LSEL(LUDATA_SEL, SEL_UPL)),TF_DS(%rsp)
1.32 bouyer 1173: #else
1174: /* Xen already switched to kernel stack */
1175: pushq %rsi
1.36 dsl 1176: STI(si)
1.32 bouyer 1177: popq %rsi
1.37 dsl 1178: addq $0x10,%rsp /* gap to match cs:rip */
1179: pushq $2 /* error code */
1.32 bouyer 1180: pushq $T_ASTFLT
1.37 dsl 1181: subq $TF_REGSIZE,%rsp
1.32 bouyer 1182: INTR_SAVE_GPRS
1183: movw %fs,TF_FS(%rsp)
1184: movw %gs,TF_GS(%rsp)
1185: movw %es,TF_ES(%rsp)
1186: movw $(LSEL(LUDATA_SEL, SEL_UPL)),TF_DS(%rsp)
1187: #endif
1.1 fvdl 1188:
1.70 dsl 1189: do_syscall:
1.1 fvdl 1190: movq CPUVAR(CURLWP),%r14
1.64 matt 1191: incq CPUVAR(NSYSCALL) # count it atomically
1.1 fvdl 1192: movq %rsp,L_MD_REGS(%r14) # save pointer to frame
1193: movq L_PROC(%r14),%r15
1.70 dsl 1194: andl $~MDL_IRET,L_MD_FLAGS(%r14) /* Allow sysret return */
1.37 dsl 1195: movq %rsp,%rdi /* Pass frame as arg0 */
1.1 fvdl 1196: call *P_MD_SYSCALL(%r15)
1.29 yamt 1197: .Lsyscall_checkast:
1.70 dsl 1198: /*
1199: * Disable interrupts to avoid new ASTs (etc) being added and
1200: * to ensure we don't take an interrupt with some of the user
1201: * registers loaded.
1202: */
1203: CLI(si)
1.29 yamt 1204: /* Check for ASTs on exit to user mode. */
1.41 ad 1205: movl L_MD_ASTPENDING(%r14), %eax
1206: orl CPUVAR(WANT_PMAPLOAD), %eax
1.29 yamt 1207: jnz 9f
1.1 fvdl 1208: #ifdef DIAGNOSTIC
1209: cmpl $IPL_NONE,CPUVAR(ILEVEL)
1210: jne 3f
1211: #endif
1.70 dsl 1212: testl $(MDL_IRET | MDL_COMPAT32), L_MD_FLAGS(%r14)
1213: INTR_RESTORE_GPRS
1.58 chs 1214: movw TF_ES(%rsp),%es
1.70 dsl 1215: SWAPGS
1216: jnz 2f
1.32 bouyer 1217: #ifndef XEN
1.70 dsl 1218: movq TF_RIP(%rsp), %rcx /* %rip for sysret */
1219: movq TF_RFLAGS(%rsp), %r11 /* %flags for sysret */
1220: movw TF_DS(%rsp), %ds
1221: movq TF_RSP(%rsp), %rsp
1.1 fvdl 1222: sysretq
1.32 bouyer 1223: #else
1.70 dsl 1224: movw TF_DS(%rsp), %ds
1225: addq $TF_RIP, %rsp
1.32 bouyer 1226: pushq $256 /* VGCF_IN_SYSCALL */
1227: jmp HYPERVISOR_iret
1228: #endif
1.1 fvdl 1229:
1.70 dsl 1230: /*
1231: * If the syscall might have modified some registers, or we are a 32bit
1232: * process we must return to user with an 'iret' instruction.
1233: * If the iret faults in kernel (assumed due to illegal register values)
1234: * then a SIGSEGV will be signalled.
1235: */
1236: 2:
1237: movw TF_DS(%rsp), %ds
1238: addq $TF_RIP, %rsp
1239: iretq
1240:
1241: /* Report SPL error */
1.1 fvdl 1242: #ifdef DIAGNOSTIC
1243: 3: movabsq $4f, %rdi
1244: movl TF_RAX(%rsp),%esi
1245: movl TF_RDI(%rsp),%edx
1246: movl %ebx,%ecx
1247: movl CPUVAR(ILEVEL),%r8d
1248: xorq %rax,%rax
1249: call _C_LABEL(printf)
1.35 ad 1250: movl $IPL_NONE,%edi
1251: call _C_LABEL(spllower)
1.29 yamt 1252: jmp .Lsyscall_checkast
1.1 fvdl 1253: 4: .asciz "WARNING: SPL NOT LOWERED ON SYSCALL %d %d EXIT %x %x\n"
1254: #endif
1.70 dsl 1255:
1256: /* AST pending or pmap load needed */
1.41 ad 1257: 9:
1258: cmpl $0, CPUVAR(WANT_PMAPLOAD)
1259: jz 10f
1260: STI(si)
1.29 yamt 1261: call _C_LABEL(do_pmap_load)
1262: jmp .Lsyscall_checkast /* re-check ASTs */
1.41 ad 1263: 10:
1264: CLEAR_ASTPENDING(%r14)
1265: STI(si)
1266: /* Pushed T_ASTFLT into tf_trapno on entry. */
1267: movq %rsp,%rdi
1268: call _C_LABEL(trap)
1269: jmp .Lsyscall_checkast /* re-check ASTs */
1.71 uebayasi 1270: IDTVEC_END(syscall)
1.29 yamt 1271:
1.22 yamt 1272: /*
1273: * void lwp_trampoline(void);
1274: *
1275: * This is a trampoline function pushed run by newly created LWPs
1.70 dsl 1276: * in order to do additional setup in their context.
1.22 yamt 1277: */
1278: NENTRY(lwp_trampoline)
1279: movq %rbp,%rsi
1.41 ad 1280: movq %rbp,%r14 /* for .Lsyscall_checkast */
1.22 yamt 1281: movq %rax,%rdi
1282: xorq %rbp,%rbp
1283: call _C_LABEL(lwp_startup)
1.1 fvdl 1284: movq %r13,%rdi
1285: call *%r12
1.41 ad 1286: jmp .Lsyscall_checkast
1.71 uebayasi 1287: END(lwp_trampoline)
1.1 fvdl 1288:
1289: /*
1.22 yamt 1290: * osyscall()
1291: *
1.1 fvdl 1292: * Trap gate entry for int $80 syscall, also used by sigreturn.
1293: */
1294: IDTVEC(osyscall)
1.32 bouyer 1295: #ifdef XEN
1296: movq (%rsp),%rcx
1297: movq 8(%rsp),%r11
1298: addq $0x10,%rsp
1299: #endif
1.1 fvdl 1300: pushq $2 # size of instruction for restart
1301: pushq $T_ASTFLT # trap # for doing ASTs
1302: INTRENTRY
1.36 dsl 1303: STI(si)
1.70 dsl 1304: jmp do_syscall
1.71 uebayasi 1305: IDTVEC_END(osyscall)
1.26 ad 1306:
1307: /*
1.66 chs 1308: * bool sse2_idlezero_page(void *pg)
1.26 ad 1309: *
1.47 ad 1310: * Zero a page without polluting the cache. Preemption must be
1311: * disabled by the caller. Abort if a preemption is pending.
1.66 chs 1312: * Returns true if the page is zeroed, false if not.
1.26 ad 1313: */
1.47 ad 1314: ENTRY(sse2_idlezero_page)
1315: pushq %rbp
1316: movq %rsp,%rbp
1317: movl $(PAGE_SIZE/64), %ecx
1.26 ad 1318: xorq %rax, %rax
1319: .align 16
1320: 1:
1.50 ad 1321: testl $RESCHED_KPREEMPT, CPUVAR(RESCHED)
1.47 ad 1322: jnz 2f
1.26 ad 1323: movnti %rax, 0(%rdi)
1324: movnti %rax, 8(%rdi)
1325: movnti %rax, 16(%rdi)
1326: movnti %rax, 24(%rdi)
1327: movnti %rax, 32(%rdi)
1328: movnti %rax, 40(%rdi)
1329: movnti %rax, 48(%rdi)
1330: movnti %rax, 56(%rdi)
1.47 ad 1331: addq $64, %rdi
1332: decl %ecx
1.26 ad 1333: jnz 1b
1334: sfence
1.47 ad 1335: incl %eax
1336: popq %rbp
1.26 ad 1337: ret
1.47 ad 1338: 2:
1.26 ad 1339: sfence
1.47 ad 1340: popq %rbp
1.26 ad 1341: ret
1.71 uebayasi 1342: END(sse2_idlezero_page)
1.66 chs 1343:
1344: /*
1345: * void pagezero(vaddr_t va)
1346: *
1347: * Zero a page without polluting the cache.
1348: */
1349:
1350: ENTRY(pagezero)
1351: movq $-PAGE_SIZE,%rdx
1352: subq %rdx,%rdi
1353: xorq %rax,%rax
1354: 1:
1355: movnti %rax,(%rdi,%rdx)
1356: movnti %rax,8(%rdi,%rdx)
1357: movnti %rax,16(%rdi,%rdx)
1358: movnti %rax,24(%rdi,%rdx)
1359: movnti %rax,32(%rdi,%rdx)
1360: movnti %rax,40(%rdi,%rdx)
1361: movnti %rax,48(%rdi,%rdx)
1362: movnti %rax,56(%rdi,%rdx)
1363: addq $64,%rdx
1364: jne 1b
1365: sfence
1366: ret
1.71 uebayasi 1367: END(pagezero)
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to locore.S CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / amd64 / amd64